From 2d216a11f8c2ed125e46de581ed255bc138fffe1 Mon Sep 17 00:00:00 2001 From: chaijiacheng888 Date: Sun, 20 Sep 2026 14:17:21 +0800 Subject: [PATCH] [Model] Serve DeepSeek-OCR-2 with its official 768px local-crop geometry (#38996) --- python/sglang/srt/configs/deepseek_ocr.py | 44 ++++++++-- python/sglang/srt/models/deepseek_ocr.py | 14 +--- .../srt/multimodal/processors/deepseek_ocr.py | 24 ++++-- .../multimodal/test_deepseek_ocr_geometry.py | 82 +++++++++++++++++++ 4 files changed, 140 insertions(+), 24 deletions(-) create mode 100644 test/registered/unit/multimodal/test_deepseek_ocr_geometry.py diff --git a/python/sglang/srt/configs/deepseek_ocr.py b/python/sglang/srt/configs/deepseek_ocr.py index 3cde7cc86..757b9473b 100644 --- a/python/sglang/srt/configs/deepseek_ocr.py +++ b/python/sglang/srt/configs/deepseek_ocr.py @@ -23,7 +23,8 @@ from sglang.srt.sampling.custom_logit_processor import ( DeepseekOCRImage = Union[Image.Image, torch.Tensor] BASE_SIZE = 1024 -IMAGE_SIZE = 640 +IMAGE_SIZE = 640 # DeepSeek-OCR local crop; OCR-2 uses OCR2_IMAGE_SIZE +OCR2_IMAGE_SIZE = 768 CROP_MODE = True MIN_CROPS = 2 MAX_CROPS = 6 # max:9; If your GPU memory is small, it is recommended to set it to 6. @@ -218,7 +219,11 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_ def dynamic_preprocess( - image, min_num=MIN_CROPS, max_num=MAX_CROPS, image_size=640, use_thumbnail=False + image, + min_num=MIN_CROPS, + max_num=MAX_CROPS, + image_size=IMAGE_SIZE, + use_thumbnail=False, ): orig_width, orig_height = get_image_size(image) aspect_ratio = orig_width / orig_height @@ -263,6 +268,28 @@ def dynamic_preprocess( return processed_images, target_aspect_ratio +def is_ocr2_config(config) -> bool: + """Whether a checkpoint is DeepSeek-OCR-2. + + Both checkpoints ship identical processor configs, so identity comes from the + model config: the DeepEncoder V2 vision encoder, or its 896-dim projector. + Both lookups are guarded because the projector clause is what covers derived + checkpoints that drop `model_name` -- reading it unguarded would raise before + that clause is reached. + """ + vision_config = getattr(config, "vision_config", None) + projector_config = getattr(config, "projector_config", None) + return ( + str(getattr(vision_config, "model_name", "")).lower() == "deepencoderv2" + or getattr(projector_config, "input_dim", None) == 896 + ) + + +def local_crop_size(config) -> int: + """Local-crop pixel size: 768 for OCR-2, 640 for DeepSeek-OCR.""" + return OCR2_IMAGE_SIZE if is_ocr2_config(config) else IMAGE_SIZE + + class DeepseekOCRProcessor(ProcessorMixin): tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") attributes = ["tokenizer"] @@ -287,6 +314,8 @@ class DeepseekOCRProcessor(ProcessorMixin): ): self.candidate_resolutions = candidate_resolutions + # The checkpoint config carries the *global* base here, not the local + # crop; the SGLang processor patches the real crop size per model. self.image_size = candidate_resolutions[0][0] self.patch_size = patch_size self.image_mean = image_mean @@ -543,18 +572,23 @@ class DeepseekOCRProcessor(ProcessorMixin): img_w, img_h = get_image_size(image) image_shapes.append((img_w, img_h)) - if img_w <= 640 and img_h <= 640: + # Both official processors threshold on their own crop size (640 for + # OCR-1, 768 for OCR-2), which is what `image_size` holds here. + if img_w <= self.image_size and img_h <= self.image_size: crop_ratio = [1, 1] else: if cropping: images_crop_raw, crop_ratio = dynamic_preprocess( - image, image_size=IMAGE_SIZE + image, image_size=self.image_size ) else: crop_ratio = [1, 1] """process the global view""" - if self.image_size <= 640 and not cropping: + # Upstream compares against the model's own crop constant, which is + # exactly what `image_size` holds, so the test is always true and the + # guard reduces to `not cropping`. + if not cropping: image = resize_image(image, (self.image_size, self.image_size)) global_view = pad_image( diff --git a/python/sglang/srt/models/deepseek_ocr.py b/python/sglang/srt/models/deepseek_ocr.py index 0f7d640a2..683032e93 100644 --- a/python/sglang/srt/models/deepseek_ocr.py +++ b/python/sglang/srt/models/deepseek_ocr.py @@ -31,7 +31,7 @@ import transformers from torch import Tensor, nn from transformers.models.vitdet.modeling_vitdet import get_rel_pos -from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config +from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config, is_ocr2_config from sglang.srt.layers.quantization import QuantizationConfig from sglang.srt.managers.mm_utils import ( MultiModalityDataPaddingPatternMultimodalTokens, @@ -1420,18 +1420,12 @@ def build_qwen2_decoder_as_encoder( return decoder_as_encoder -def _is_ocr2(config: DeepseekVLV2Config) -> bool: - return ( - str(getattr(config.vision_config, "model_name", "")).lower() == "deepencoderv2" - or getattr(config.projector_config, "input_dim", None) == 896 - ) - - class DeepseekOCRForCausalLM(nn.Module): @staticmethod def shared_experts_fusion_disable_reason(hf_config, quant_config): text_config = hf_config.text_config - if _is_ocr2(hf_config) or not ( + # Class-level hook: called before the model is built, so no `self.is_ocr2` yet. + if is_ocr2_config(hf_config) or not ( text_config.topk_method == "noaux_tc" or text_config.use_mla ): # Those branches build the dense DeepseekForCausalLM, which has no @@ -1455,7 +1449,7 @@ class DeepseekOCRForCausalLM(nn.Module): self.vision_config = config.vision_config self.projector_config = config.projector_config self.text_config = config.text_config - self.is_ocr2 = _is_ocr2(config) + self.is_ocr2 = is_ocr2_config(config) n_embed = getattr(self.projector_config, "n_embed", 1280) self.tile_tag = config.tile_tag diff --git a/python/sglang/srt/multimodal/processors/deepseek_ocr.py b/python/sglang/srt/multimodal/processors/deepseek_ocr.py index dbb06c0d1..0086680aa 100644 --- a/python/sglang/srt/multimodal/processors/deepseek_ocr.py +++ b/python/sglang/srt/multimodal/processors/deepseek_ocr.py @@ -1,5 +1,6 @@ from typing import List, Union +from sglang.srt.configs.deepseek_ocr import is_ocr2_config, local_crop_size from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput from sglang.srt.models.deepseek_ocr import DeepseekOCRForCausalLM from sglang.srt.multimodal.processors.base_processor import ( @@ -8,19 +9,24 @@ from sglang.srt.multimodal.processors.base_processor import ( ) +def apply_ocr_geometry(processor, hf_config) -> None: + """Patch a checkpoint's local-crop geometry onto an already-built HF processor. + + `image_size` is overwritten unconditionally: neither checkpoint carries the + crop size (see `local_crop_size`), so a value in `processor_config.json` would + not survive. + """ + processor.ocr2_mode = is_ocr2_config(hf_config) + processor.image_size = local_crop_size(hf_config) + + class DeepseekOCRProcessor(BaseMultimodalProcessor): models = [DeepseekOCRForCausalLM] def __init__(self, hf_config, server_args, _processor, *args, **kwargs): - _processor.image_size = 640 - _processor.ocr2_mode = ( - str( - getattr(getattr(hf_config, "vision_config", None), "model_name", "") - ).lower() - == "deepencoderv2" - or getattr(getattr(hf_config, "projector_config", None), "input_dim", None) - == 896 - ) + # The shared processor config's candidate_resolutions is the *global* base, so + # the crop size has to come from the model identity instead. + apply_ocr_geometry(_processor, hf_config) super().__init__(hf_config, server_args, _processor, *args, **kwargs) self.mm_tokens = MultimodalSpecialTokens( image_token="", image_token_id=self._processor.image_token_id diff --git a/test/registered/unit/multimodal/test_deepseek_ocr_geometry.py b/test/registered/unit/multimodal/test_deepseek_ocr_geometry.py new file mode 100644 index 000000000..6dba0e043 --- /dev/null +++ b/test/registered/unit/multimodal/test_deepseek_ocr_geometry.py @@ -0,0 +1,82 @@ +"""DeepSeek-OCR-2 must run 768px local crops, not the 640px ones inherited from +DeepSeek-OCR: a 768px crop expands to (768 // 16 // 4) ** 2 == 144 visual tokens, +exactly the length of the tuned query_768 table in Qwen2Decoder2Encoder, while +640px yields 100 and falls back to interpolation. + +Neither checkpoint carries the crop size -- both ship identical processor configs +whose candidate_resolutions=[[1024, 1024]] is the *global* base -- so it is +selected from the model config and patched onto the processor. The in-processor +thresholds that consume `image_size` need a real tokenizer, so they are only +observable end to end. +""" + +import unittest +from types import SimpleNamespace + +from PIL import Image + +from sglang.srt.configs.deepseek_ocr import ( + IMAGE_SIZE, + OCR2_IMAGE_SIZE, + dynamic_preprocess, + local_crop_size, +) +from sglang.srt.multimodal.processors.deepseek_ocr import apply_ocr_geometry +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=60, suite="base-a-test-cpu") + + +def _hf_config(vision_model_name: str, projector_input_dim) -> SimpleNamespace: + return SimpleNamespace( + vision_config=SimpleNamespace(model_name=vision_model_name), + projector_config=SimpleNamespace(input_dim=projector_input_dim), + ) + + +def _ocr2_config() -> SimpleNamespace: + """DeepSeek-OCR-2: DeepEncoder V2 vision encoder, 896-dim projector.""" + return _hf_config("deepencoderv2", 896) + + +def _ocr1_config() -> SimpleNamespace: + """DeepSeek-OCR: deeplip_b_l vision encoder, 2048-dim projector.""" + return _hf_config("deeplip_b_l", 2048) + + +class TestDeepseekOcrGeometry(CustomTestCase): + def test_local_crop_size_is_768_for_ocr2_and_640_for_ocr1(self): + # OCR-2 must not be served with OCR-1's 640px crops; OCR-1 must stay on 640. + self.assertEqual(local_crop_size(_ocr2_config()), 768) + self.assertEqual(local_crop_size(_ocr2_config()), OCR2_IMAGE_SIZE) + self.assertEqual(local_crop_size(_ocr1_config()), 640) + self.assertEqual(local_crop_size(_ocr1_config()), IMAGE_SIZE) + + def test_processor_geometry_wiring(self): + # Assert the patching itself, so a regression in the call site is caught too. + stub = SimpleNamespace() + apply_ocr_geometry(stub, _ocr2_config()) + self.assertEqual(stub.image_size, 768) + self.assertTrue(stub.ocr2_mode) + apply_ocr_geometry(stub, _ocr1_config()) + self.assertEqual(stub.image_size, 640) + self.assertFalse(stub.ocr2_mode) + + def test_dynamic_preprocess_crop_px_follows_image_size(self): + # `tokenize_with_images` passes `image_size` through, so the crop pixels + # follow whichever crop size was selected above. + wide = Image.new("RGB", (3072, 1024)) # aspect 3:1 -> 3x1 tile grid + for image_size, expected_size in ((768, 768), (640, 640)): + crops, ratio = dynamic_preprocess( + image=wide, image_size=image_size, min_num=2, max_num=6 + ) + self.assertEqual(ratio, (3, 1)) + self.assertEqual(len(crops), 3) + self.assertTrue( + all(crop.size == (expected_size, expected_size) for crop in crops) + ) + + +if __name__ == "__main__": + unittest.main()