[Fix] mistral3: Subclass LlavaForConditionalGeneration instead of wrapping it (#45656)

This commit is contained in:
em-dreyer
2026-10-23 14:05:32 -07:00
committed by GitHub
parent 194abb8e54
commit 4542fc8f59
3 changed files with 125 additions and 27 deletions
+3 -1
View File
@@ -47,7 +47,6 @@ from sglang.srt.managers.schedule_batch import (
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.llama import LlamaForCausalLM
from sglang.srt.models.mistral import MistralForCausalLM
from sglang.srt.models.qwen2 import Qwen2ForCausalLM
from sglang.srt.multimodal.mm_utils import (
get_anyres_image_grid_shape,
@@ -604,6 +603,9 @@ class LlavaMistralForCausalLM(LlavaBaseForCausalLM):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
# mistral.py subclasses LlavaForConditionalGeneration from this module
from sglang.srt.models.mistral import MistralForCausalLM
super().__init__()
self.config = config
+15 -25
View File
@@ -15,14 +15,17 @@
import logging
from collections.abc import Iterable
from typing import List
from typing import List, Optional
import regex as re
import torch
from transformers import LlavaConfig
from transformers.models.mistral3.modeling_mistral3 import Mistral3MultiModalProjector
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.managers.schedule_batch import MultimodalDataItem
from sglang.srt.models.llama import LlamaForCausalLM
from sglang.srt.models.llava import LlavaForConditionalGeneration
logger = logging.getLogger(__name__)
@@ -88,22 +91,18 @@ class MistralForCausalLMMistralFormat(MistralForCausalLM):
yield name, loaded_weight
class Mistral3ForConditionalGeneration:
class Mistral3ForConditionalGeneration(LlavaForConditionalGeneration):
MULTIMODAL_PROJECTOR_TYPE = Mistral3MultiModalProjector
def __init__(self, **kwargs):
# lazy load inner class
# to bypass circular import
from sglang.srt.models.llava import LlavaForConditionalGeneration
# override config: mistral's projector adds patchmerger that doesn't require padding
kwargs["config"].vision_config.pad_image_border = False
self.inner = LlavaForConditionalGeneration(**kwargs)
self.inner.multi_modal_projector = self.MULTIMODAL_PROJECTOR_TYPE(
kwargs["config"]
)
self.inner.get_image_feature = self.get_image_feature
def __init__(
self,
config: LlavaConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
# mistral's projector adds a patch merger that doesn't require padding
config.vision_config.pad_image_border = False
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
"""Extract features from image inputs.
@@ -148,15 +147,6 @@ class Mistral3ForConditionalGeneration:
ret = torch.cat(features, dim=0)
return ret
def __getattr__(self, name):
return getattr(self.inner, name)
def __hasattr__(self, name):
return hasattr(self.inner, name)
def __call__(self, *args, **kwargs):
return self.inner(*args, **kwargs)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
"""Normalize transformers v5 Mistral3 weight names for
LlavaForConditionalGeneration.load_weights.
@@ -195,7 +185,7 @@ class Mistral3ForConditionalGeneration:
name = "language_model." + name
yield name, w
return self.inner.load_weights(normalize(weights))
return super().load_weights(normalize(weights))
EntryClass = [MistralForCausalLM, Mistral3ForConditionalGeneration]