diff --git a/python/sglang/srt/models/llava.py b/python/sglang/srt/models/llava.py index d6b19d1814..65eb29000f 100644 --- a/python/sglang/srt/models/llava.py +++ b/python/sglang/srt/models/llava.py @@ -47,7 +47,6 @@ from sglang.srt.managers.schedule_batch import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.llama import LlamaForCausalLM -from sglang.srt.models.mistral import MistralForCausalLM from sglang.srt.models.qwen2 import Qwen2ForCausalLM from sglang.srt.multimodal.mm_utils import ( get_anyres_image_grid_shape, @@ -604,6 +603,9 @@ class LlavaMistralForCausalLM(LlavaBaseForCausalLM): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", ) -> None: + # mistral.py subclasses LlavaForConditionalGeneration from this module + from sglang.srt.models.mistral import MistralForCausalLM + super().__init__() self.config = config diff --git a/python/sglang/srt/models/mistral.py b/python/sglang/srt/models/mistral.py index dfd9c17e88..9d9eb237fe 100644 --- a/python/sglang/srt/models/mistral.py +++ b/python/sglang/srt/models/mistral.py @@ -15,14 +15,17 @@ import logging from collections.abc import Iterable -from typing import List +from typing import List, Optional import regex as re import torch +from transformers import LlavaConfig from transformers.models.mistral3.modeling_mistral3 import Mistral3MultiModalProjector +from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.managers.schedule_batch import MultimodalDataItem from sglang.srt.models.llama import LlamaForCausalLM +from sglang.srt.models.llava import LlavaForConditionalGeneration logger = logging.getLogger(__name__) @@ -88,22 +91,18 @@ class MistralForCausalLMMistralFormat(MistralForCausalLM): yield name, loaded_weight -class Mistral3ForConditionalGeneration: +class Mistral3ForConditionalGeneration(LlavaForConditionalGeneration): MULTIMODAL_PROJECTOR_TYPE = Mistral3MultiModalProjector - def __init__(self, **kwargs): - # lazy load inner class - # to bypass circular import - from sglang.srt.models.llava import LlavaForConditionalGeneration - - # override config: mistral's projector adds patchmerger that doesn't require padding - kwargs["config"].vision_config.pad_image_border = False - - self.inner = LlavaForConditionalGeneration(**kwargs) - self.inner.multi_modal_projector = self.MULTIMODAL_PROJECTOR_TYPE( - kwargs["config"] - ) - self.inner.get_image_feature = self.get_image_feature + def __init__( + self, + config: LlavaConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + # mistral's projector adds a patch merger that doesn't require padding + config.vision_config.pad_image_border = False + super().__init__(config=config, quant_config=quant_config, prefix=prefix) def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: """Extract features from image inputs. @@ -148,15 +147,6 @@ class Mistral3ForConditionalGeneration: ret = torch.cat(features, dim=0) return ret - def __getattr__(self, name): - return getattr(self.inner, name) - - def __hasattr__(self, name): - return hasattr(self.inner, name) - - def __call__(self, *args, **kwargs): - return self.inner(*args, **kwargs) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): """Normalize transformers v5 Mistral3 weight names for LlavaForConditionalGeneration.load_weights. @@ -195,7 +185,7 @@ class Mistral3ForConditionalGeneration: name = "language_model." + name yield name, w - return self.inner.load_weights(normalize(weights)) + return super().load_weights(normalize(weights)) EntryClass = [MistralForCausalLM, Mistral3ForConditionalGeneration] diff --git a/test/registered/unit/models/test_mistral3_vision_feature.py b/test/registered/unit/models/test_mistral3_vision_feature.py index c7df5d06b0..2cf9596540 100644 --- a/test/registered/unit/models/test_mistral3_vision_feature.py +++ b/test/registered/unit/models/test_mistral3_vision_feature.py @@ -6,6 +6,8 @@ final-layer case takes the cheap path and that both paths agree, for each adaptor that reads the Pixtral tower this way. """ +import subprocess +import sys import unittest from types import SimpleNamespace @@ -15,7 +17,7 @@ from sglang.srt.models.llava import LlavaForConditionalGeneration from sglang.srt.models.mistral import Mistral3ForConditionalGeneration from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu") +register_cpu_ci(est_time=40, suite="base-a-test-cpu") NUM_LAYERS = 4 TOKENS = 3 @@ -105,5 +107,109 @@ class TestPixtralAdaptorVisionFeature(unittest.TestCase): self.assertEqual(out.shape, (3 * TOKENS, HIDDEN)) +def _bare_mistral3(): + # Skip LlavaForConditionalGeneration.__init__, which builds the language and + # vision models from an HF config. + model = Mistral3ForConditionalGeneration.__new__(Mistral3ForConditionalGeneration) + torch.nn.Module.__init__(model) + return model + + +class RecordingLoader(torch.nn.Module): + def __init__(self): + super().__init__() + self.loaded = [] + + def load_weights(self, weights): + self.loaded.extend(name for name, _ in weights) + + +class RecordingProjector(torch.nn.Module): + # Mistral3's projector takes image_sizes; Llava's takes the feature alone. + def __init__(self): + super().__init__() + self.image_sizes = [] + + def forward(self, feature, image_sizes): + self.image_sizes.append(image_sizes) + return feature + + +class TestMistral3ModelSurvivesEval(unittest.TestCase): + def test_eval_keeps_the_mistral3_model(self): + """Loaders keep what ``model.eval()`` returns, so it must be the model the + loader built: same ``shared_experts_config``, children in eval mode, and + Mistral3's image-feature path.""" + model = _bare_mistral3() + sentinel = object() + model.shared_experts_config = sentinel + model.vision_tower = RecordingTower() + model.vision_feature_layer = -1 + model.vision_feature_select_strategy = "full" + model.multi_modal_projector = RecordingProjector() + + loaded = model.eval() + + self.assertIs(loaded, model) + self.assertIs(loaded.shared_experts_config, sentinel) + self.assertFalse(loaded.multi_modal_projector.training) + loaded.get_image_feature(_items(1)) + self.assertEqual(loaded.multi_modal_projector.image_sizes, [[(4, 4)]]) + + def test_v5_checkpoint_names_load_the_same_before_and_after_eval(self): + """Weight updates call ``load_weights`` on the model kept after ``eval()``, + so transformers v5 checkpoint names must reach the same destinations as + the initial load.""" + model = _bare_mistral3() + model.vision_feature_select_strategy = "full" + model.language_model = RecordingLoader() + model.vision_tower = RecordingLoader() + model.multi_modal_projector = torch.nn.Linear(2, 2, bias=False) + projector_weight = torch.ones(2, 2) + v5_weights = [ + ("model.language_model.embed_tokens.weight", torch.zeros(1)), + ("lm_head.weight", torch.zeros(1)), + ("model.vision_tower.patch_conv.weight", torch.zeros(1)), + ("model.multi_modal_projector.weight", projector_weight), + ] + + def load_into(target): + model.language_model.loaded.clear() + model.vision_tower.loaded.clear() + torch.nn.init.zeros_(model.multi_modal_projector.weight) + target.load_weights(v5_weights) + return ( + list(model.language_model.loaded), + list(model.vision_tower.loaded), + torch.equal(model.multi_modal_projector.weight, projector_weight), + ) + + expected = ( + ["model.embed_tokens.weight", "lm_head.weight"], + ["patch_conv.weight"], + True, + ) + self.assertEqual(load_into(model), expected) + self.assertEqual(load_into(model.eval()), expected) + + +class TestMistralLlavaImportOrder(unittest.TestCase): + def test_either_module_can_be_imported_first(self): + """mistral.py subclasses LlavaForConditionalGeneration and llava.py builds + MistralForCausalLM, so neither module may depend on import order.""" + for first, second in (("mistral", "llava"), ("llava", "mistral")): + with self.subTest(first=first): + result = subprocess.run( + [ + sys.executable, + "-c", + f"import sglang.srt.models.{first}, sglang.srt.models.{second}", + ], + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + if __name__ == "__main__": unittest.main()