[Model] Add support for EmbeddingGemma 2 (#42782)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Luciano Martins
Claude Opus 5.5
parent
b193ef382a
commit
d1ba943016
@@ -698,10 +698,18 @@ def _fwd_kernel(
|
||||
final_mask &= mask_non_causal
|
||||
|
||||
if SLIDING_WINDOW_SIZE > 0:
|
||||
# Add mask where q_id <= kv_id + sliding_window_size
|
||||
window_mask = (cur_block_m * BLOCK_M + offs_m[:, None]) <= (
|
||||
start_n + offs_n[None, :] + SLIDING_WINDOW_SIZE
|
||||
)
|
||||
if not IS_CAUSAL:
|
||||
window_mask = (
|
||||
(cur_block_m * BLOCK_M + offs_m[:, None])
|
||||
<= (start_n + offs_n[None, :] + SLIDING_WINDOW_SIZE)
|
||||
) & (
|
||||
(start_n + offs_n[None, :])
|
||||
<= (cur_block_m * BLOCK_M + offs_m[:, None] + SLIDING_WINDOW_SIZE)
|
||||
)
|
||||
else:
|
||||
window_mask = (cur_block_m * BLOCK_M + offs_m[:, None]) <= (
|
||||
start_n + offs_n[None, :] + SLIDING_WINDOW_SIZE
|
||||
)
|
||||
final_mask &= window_mask
|
||||
|
||||
SKIP_TILE = False
|
||||
@@ -1312,6 +1320,8 @@ def _fwd_kernel_unified(
|
||||
|
||||
# Sliding window: query can attend to keys within window_size
|
||||
window_mask = q_abs_pos <= (k_abs_pos + SLIDING_WINDOW_SIZE)
|
||||
if not IS_CAUSAL:
|
||||
window_mask &= k_abs_pos <= (q_abs_pos + SLIDING_WINDOW_SIZE)
|
||||
final_mask &= window_mask
|
||||
|
||||
# Check if we can skip this tile
|
||||
|
||||
@@ -836,6 +836,24 @@ def handle_model_capability_adjustments(server_args: Any):
|
||||
logger.info(
|
||||
"Embedding architecture detected: enabling embedding mode automatically."
|
||||
)
|
||||
if (
|
||||
embedding_model_spec is not None
|
||||
and embedding_model_spec.safe_disable_radix_cache
|
||||
):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_model_capability_adjustments",
|
||||
disable_radix_cache=True,
|
||||
)
|
||||
if (
|
||||
embedding_model_spec is not None
|
||||
and embedding_model_spec.safe_disable_chunked_prefill
|
||||
):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_model_capability_adjustments",
|
||||
chunked_prefill_size=-1,
|
||||
)
|
||||
|
||||
is_embedding_gemma = (
|
||||
embedding_model_spec is not None
|
||||
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.arg_groups.model_overrides import bailing_moe_v3 # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import cohere2_moe # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import deepseek_v2 # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import deepseek_v4 # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import embedding_gemma2 # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import ernie45_vl # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import exaone # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import falcon_h1 # noqa: F401
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# Copyright 2025 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
"""Config-time override declarations for EmbeddingGemma2.
|
||||
|
||||
Architectures: EmbeddingGemma2Model.
|
||||
Sets disable_radix_cache=True and default attention_backend="triton" for EmbeddingGemma2.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import (
|
||||
_register_for,
|
||||
is_attention_backend_not_set,
|
||||
resolving_view,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@_register_for("EmbeddingGemma2Model")
|
||||
def _embedding_gemma2_overrides(server_args: Any, hf_config: Any) -> dict[str, Any]:
|
||||
cfg = resolving_view(server_args)
|
||||
overrides: dict[str, Any] = {
|
||||
"disable_radix_cache": True,
|
||||
}
|
||||
if is_attention_backend_not_set(cfg) or cfg.attention_backend is None:
|
||||
logger.info("Use triton as default attention backend for EmbeddingGemma2Model")
|
||||
overrides["attention_backend"] = "triton"
|
||||
|
||||
return overrides
|
||||
@@ -5,9 +5,10 @@ implementations, documentation, and benchmarks can consume the same contract
|
||||
without each reimplementing a partial list of embedding architectures.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Any, Sequence
|
||||
from typing import Any
|
||||
|
||||
|
||||
class EmbeddingTask(str, Enum):
|
||||
@@ -168,9 +169,37 @@ def embedding_support_matrix() -> list[dict[str, Any]]:
|
||||
**_embedding_gemma_spec().as_dict(),
|
||||
}
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"architecture": "EmbeddingGemma2Model",
|
||||
**_embedding_gemma2_spec().as_dict(),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _embedding_gemma2_spec() -> EmbeddingModelSpec:
|
||||
return EmbeddingModelSpec(
|
||||
family="embedding_gemma2",
|
||||
task=EmbeddingTask.EMBED,
|
||||
execution=EmbeddingExecution.MULTIMODAL,
|
||||
attention=AttentionPattern.BIDIRECTIONAL,
|
||||
pooling=PoolingStrategy.MEAN,
|
||||
normalize=True,
|
||||
postprocessor="model_defined",
|
||||
tokenizer_special_tokens="model_default",
|
||||
supports_dimensions=False,
|
||||
supports_token_embeddings=False,
|
||||
supports_multimodal=True,
|
||||
requires_embedding_flag=False,
|
||||
auto_enable_embedding=True,
|
||||
bidirectional_attention=True,
|
||||
bcg_prefill_policy=BCGPrefillPolicy.DEFAULT,
|
||||
safe_disable_radix_cache=True,
|
||||
safe_disable_chunked_prefill=True,
|
||||
)
|
||||
|
||||
|
||||
def _embedding_gemma_spec() -> EmbeddingModelSpec:
|
||||
return EmbeddingModelSpec(
|
||||
family="embeddinggemma",
|
||||
@@ -276,6 +305,9 @@ def resolve_embedding_model_spec(
|
||||
|
||||
architecture_set = set(architectures or ())
|
||||
|
||||
if "EmbeddingGemma2Model" in architecture_set:
|
||||
return _embedding_gemma2_spec()
|
||||
|
||||
if is_embedding_gemma:
|
||||
return _embedding_gemma_spec()
|
||||
|
||||
|
||||
@@ -1119,6 +1119,7 @@ class ModelConfig:
|
||||
"MiMoV2MTP",
|
||||
"Gemma4ForCausalLM",
|
||||
"Gemma4ForConditionalGeneration",
|
||||
"EmbeddingGemma2Model",
|
||||
"InklingForConditionalGeneration",
|
||||
"InklingForConditionalGenerationMTP",
|
||||
"Gemma4UnifiedForConditionalGeneration",
|
||||
@@ -1187,7 +1188,13 @@ class ModelConfig:
|
||||
else:
|
||||
self.context_len = context_length
|
||||
else:
|
||||
self.context_len = derived_context_len
|
||||
if (
|
||||
self.hf_config.architectures
|
||||
and self.hf_config.architectures[0] == "EmbeddingGemma2Model"
|
||||
):
|
||||
self.context_len = min(derived_context_len, 8192)
|
||||
else:
|
||||
self.context_len = derived_context_len
|
||||
|
||||
# Transfer context_len to HuggingFace config so models can access it
|
||||
self.hf_config.context_len = self.context_len
|
||||
@@ -2241,6 +2248,7 @@ def is_generation_model(model_architectures: List[str], is_embedding: bool = Fal
|
||||
or "XLMRobertaModel" in model_architectures
|
||||
or "XLMRobertaForSequenceClassification" in model_architectures
|
||||
or "Gemma2ForSequenceClassification" in model_architectures
|
||||
or "EmbeddingGemma2Model" in model_architectures
|
||||
or "Lfm2BidirectionalModel" in model_architectures
|
||||
):
|
||||
return False
|
||||
@@ -2258,6 +2266,7 @@ multimodal_model_archs = [
|
||||
"Gemma3ForConditionalGeneration",
|
||||
"Gemma3nForConditionalGeneration",
|
||||
"Gemma4ForConditionalGeneration",
|
||||
"EmbeddingGemma2Model",
|
||||
"Gemma4UnifiedForConditionalGeneration",
|
||||
"DiffusionGemmaForBlockDiffusion",
|
||||
"Glm4vForConditionalGeneration",
|
||||
@@ -2519,6 +2528,7 @@ def is_hybrid_swa_model(
|
||||
"Step3p7ForConditionalGeneration",
|
||||
"Gemma4ForCausalLM",
|
||||
"Gemma4ForConditionalGeneration",
|
||||
"EmbeddingGemma2Model",
|
||||
"Gemma4UnifiedForConditionalGeneration",
|
||||
"DiffusionGemmaForBlockDiffusion",
|
||||
"LagunaForCausalLM",
|
||||
@@ -2600,6 +2610,7 @@ def get_hybrid_layer_ids(
|
||||
elif (
|
||||
"Gemma4ForCausalLM" in model_architectures
|
||||
or "Gemma4ForConditionalGeneration" in model_architectures
|
||||
or "EmbeddingGemma2Model" in model_architectures
|
||||
or "Gemma4UnifiedForConditionalGeneration" in model_architectures
|
||||
or "DiffusionGemmaForBlockDiffusion" in model_architectures
|
||||
or "LagunaForCausalLM" in model_architectures
|
||||
|
||||
@@ -1271,10 +1271,16 @@ class EmbeddingReqInput:
|
||||
)
|
||||
|
||||
def normalize_batch_and_arguments(self):
|
||||
# at least one of text, input_ids, or image should be provided
|
||||
if self.text is None and self.input_ids is None and self.image_data is None:
|
||||
# at least one of text, input_ids, image, video, or audio should be provided
|
||||
if (
|
||||
self.text is None
|
||||
and self.input_ids is None
|
||||
and self.image_data is None
|
||||
and self.video_data is None
|
||||
and self.audio_data is None
|
||||
):
|
||||
raise ValueError(
|
||||
"At least one of text, input_ids, or image should be provided"
|
||||
"At least one of text, input_ids, image, video, or audio should be provided"
|
||||
)
|
||||
|
||||
# text and input_ids cannot be provided at the same time
|
||||
@@ -1288,18 +1294,37 @@ class EmbeddingReqInput:
|
||||
# check the batch size of text
|
||||
if self.text is not None:
|
||||
if isinstance(self.text, list):
|
||||
self.batch_size += len(self.text)
|
||||
self.batch_size = len(self.text)
|
||||
self.is_single = False
|
||||
else:
|
||||
self.batch_size += 1
|
||||
|
||||
# check the batch size of input_ids
|
||||
if self.input_ids is not None:
|
||||
self.batch_size = 1
|
||||
elif self.input_ids is not None:
|
||||
if isinstance(self.input_ids[0], list):
|
||||
self.batch_size += len(self.input_ids)
|
||||
self.batch_size = len(self.input_ids)
|
||||
self.is_single = False
|
||||
else:
|
||||
self.batch_size += 1
|
||||
self.batch_size = 1
|
||||
else:
|
||||
# Without text, a flat media list is one request carrying all items
|
||||
# (the HF processor reading); only a list of per-request lists is a batch.
|
||||
media = next(
|
||||
data
|
||||
for data in (self.image_data, self.video_data, self.audio_data)
|
||||
if data is not None
|
||||
)
|
||||
if (
|
||||
isinstance(media, list)
|
||||
and len(media) > 0
|
||||
and all(isinstance(item, list) for item in media)
|
||||
):
|
||||
self.batch_size = len(media)
|
||||
self.is_single = False
|
||||
else:
|
||||
self.batch_size = 1
|
||||
self.is_single = True
|
||||
|
||||
if not self.is_single:
|
||||
self._validate_mm_batch_lengths()
|
||||
|
||||
# Fill in default arguments
|
||||
if self.is_single:
|
||||
@@ -1325,6 +1350,19 @@ class EmbeddingReqInput:
|
||||
|
||||
self._validate_rid_uniqueness()
|
||||
|
||||
def _validate_mm_batch_lengths(self):
|
||||
# A scalar media item is broadcast to every request; a list is per-request.
|
||||
for field_name, data in (
|
||||
("image_data", self.image_data),
|
||||
("video_data", self.video_data),
|
||||
("audio_data", self.audio_data),
|
||||
):
|
||||
if isinstance(data, list) and len(data) != self.batch_size:
|
||||
raise ValueError(
|
||||
f"{field_name} has {len(data)} entries but the batch has "
|
||||
f"{self.batch_size} requests; pass one entry (or None) per request."
|
||||
)
|
||||
|
||||
def _normalize_lora_paths(self, num):
|
||||
"""Normalize LoRA paths for batch processing."""
|
||||
if self.lora_path is not None:
|
||||
@@ -1361,6 +1399,17 @@ class EmbeddingReqInput:
|
||||
if i in cache:
|
||||
return cache[i]
|
||||
|
||||
# Lengths were checked in _validate_mm_batch_lengths; scalars broadcast.
|
||||
image_item = (
|
||||
self.image_data[i] if isinstance(self.image_data, list) else self.image_data
|
||||
)
|
||||
video_item = (
|
||||
self.video_data[i] if isinstance(self.video_data, list) else self.video_data
|
||||
)
|
||||
audio_item = (
|
||||
self.audio_data[i] if isinstance(self.audio_data, list) else self.audio_data
|
||||
)
|
||||
|
||||
if self.is_cross_encoder_request:
|
||||
sub = EmbeddingReqInput(
|
||||
rid=self.rid[i],
|
||||
@@ -1390,9 +1439,9 @@ class EmbeddingReqInput:
|
||||
rid=self.rid[i],
|
||||
text=self.text[i] if self.text is not None else None,
|
||||
input_ids=self.input_ids[i] if self.input_ids is not None else None,
|
||||
image_data=self.image_data[i] if self.image_data is not None else None,
|
||||
video_data=self.video_data[i] if self.video_data is not None else None,
|
||||
audio_data=self.audio_data[i] if self.audio_data is not None else None,
|
||||
image_data=image_item,
|
||||
video_data=video_item,
|
||||
audio_data=audio_item,
|
||||
embed_override_token_id=self.embed_override_token_id,
|
||||
embed_overrides=(
|
||||
self.embed_overrides[i]
|
||||
|
||||
@@ -987,7 +987,11 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
Tuple[List[int], Optional[List[int]]],
|
||||
Tuple[List[List[int]], Optional[List[List[int]]]],
|
||||
]:
|
||||
if not texts or self.tokenizer is None:
|
||||
if (
|
||||
texts is None
|
||||
or (isinstance(texts, list) and len(texts) == 0)
|
||||
or self.tokenizer is None
|
||||
):
|
||||
raise ValueError("texts cannot be empty and tokenizer must be initialized")
|
||||
|
||||
# Step 1: Detect input format and prepare for tokenization
|
||||
|
||||
@@ -0,0 +1,963 @@
|
||||
# Copyright 2025 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
"""EmbeddingGemma2 model (EmbeddingGemma v2) for SGLang.
|
||||
|
||||
Inference-only multimodal bidirectional embedding model compatible with HuggingFace.
|
||||
Supports text, image, audio, and video inputs with mean pooling and unit L2 normalization.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable
|
||||
from typing import Any, Callable, cast
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.activation import get_act_fn
|
||||
from sglang.srt.layers.attention.vision import VisionSdpaAttention
|
||||
from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import AttentionType, RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
ForwardBatch,
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalInputs,
|
||||
flatten_nested_list,
|
||||
)
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.gemma4_audio import Gemma4AudioEncoder
|
||||
from sglang.srt.models.gemma4_causal import Gemma4MLP
|
||||
from sglang.srt.models.gemma4_mm import (
|
||||
Gemma4ForConditionalGeneration,
|
||||
Gemma4MultimodalEmbedder,
|
||||
)
|
||||
from sglang.srt.models.gemma4_vision import Gemma4VisionEncoder
|
||||
from sglang.srt.runtime_context import get_mm, get_parallel
|
||||
from sglang.srt.utils import add_prefix
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EmbeddingGemma2RMSNorm(nn.Module):
|
||||
"""Root Mean Square Layer Normalization with optional learnable scale.
|
||||
|
||||
Used across EmbeddingGemma2 attention norms, projection norms, and decoder layer norms.
|
||||
Supports forward_native binding for 3D/4D tensors and with_scale=False operations
|
||||
(e.g., v_norm in self-attention layers where scaling is omitted).
|
||||
"""
|
||||
|
||||
def __init__(self, dim: int, eps: float = 1e-6, with_scale: bool = True):
|
||||
super().__init__()
|
||||
self.eps = eps
|
||||
self.with_scale = with_scale
|
||||
|
||||
if self.with_scale:
|
||||
self.weight = nn.Parameter(torch.ones(dim), requires_grad=True)
|
||||
|
||||
def _norm(self, hidden_states: torch.Tensor):
|
||||
mean_squared = hidden_states.pow(2).mean(-1, keepdim=True) + self.eps
|
||||
return hidden_states * torch.pow(mean_squared, -0.5)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
normed_output = self._norm(hidden_states.float())
|
||||
if self.with_scale:
|
||||
normed_output = normed_output * self.weight.float()
|
||||
return normed_output.type_as(hidden_states)
|
||||
|
||||
def forward_native(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.forward(hidden_states)
|
||||
|
||||
|
||||
class EmbeddingGemma2Attention(nn.Module):
|
||||
"""Attention layer for EmbeddingGemma2.
|
||||
|
||||
Implements sliding window attention with an inclusive bidirectional window (<= 512)
|
||||
and full global attention depending on config.layer_types. In sliding layers,
|
||||
each token attends to all tokens within distance <= 512 both backward and forward,
|
||||
unlike causal SWA where attention is restricted to past tokens.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
layer_idx: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.layer_idx = layer_idx
|
||||
layer_type = config.layer_types[layer_idx]
|
||||
self.is_sliding = layer_type == "sliding_attention"
|
||||
|
||||
# Dims depend on layer type
|
||||
if self.is_sliding:
|
||||
self.head_dim = getattr(config, "swa_head_dim", 256)
|
||||
self.num_kv_heads = getattr(config, "swa_num_key_value_heads", 2)
|
||||
rope_theta = config.rope_parameters["sliding_attention"].get(
|
||||
"rope_theta"
|
||||
) or config.rope_parameters["sliding_attention"].get("base", 10000.0)
|
||||
self.sliding_window = 512
|
||||
else:
|
||||
self.head_dim = getattr(config, "head_dim", 512)
|
||||
self.num_kv_heads = getattr(config, "num_key_value_heads", 1)
|
||||
rope_theta = config.rope_parameters["full_attention"].get(
|
||||
"rope_theta"
|
||||
) or config.rope_parameters["full_attention"].get("base", 1000000.0)
|
||||
self.sliding_window = -1
|
||||
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
self.total_num_kv_heads = self.num_kv_heads
|
||||
tp_size = get_parallel().tp_size
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
config.hidden_size,
|
||||
self.head_dim,
|
||||
self.total_num_heads,
|
||||
self.total_num_kv_heads,
|
||||
bias=getattr(config, "attention_bias", False),
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
config.hidden_size,
|
||||
bias=getattr(config, "attention_bias", False),
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
self.q_norm = EmbeddingGemma2RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = EmbeddingGemma2RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.v_norm = EmbeddingGemma2RMSNorm(
|
||||
self.head_dim, eps=config.rms_norm_eps, with_scale=False
|
||||
)
|
||||
|
||||
max_pos = min(
|
||||
config.max_position_embeddings,
|
||||
getattr(config, "context_len", config.max_position_embeddings),
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
rotary_dim=self.head_dim,
|
||||
max_position=max_pos,
|
||||
base=int(rope_theta),
|
||||
is_neox_style=True,
|
||||
)
|
||||
|
||||
self.attn = RadixAttention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
scaling=1.0,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
layer_id=layer_idx,
|
||||
sliding_window_size=self.sliding_window,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("attn", prefix),
|
||||
attn_type=AttentionType.ENCODER_ONLY,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
q = q.unflatten(-1, (self.num_heads, self.head_dim))
|
||||
q = self.q_norm(q).flatten(-2, -1)
|
||||
|
||||
k = k.unflatten(-1, (self.num_kv_heads, self.head_dim))
|
||||
k = self.k_norm(k).flatten(-2, -1)
|
||||
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
q = q.unflatten(-1, (self.num_heads, self.head_dim))
|
||||
k = k.unflatten(-1, (self.num_kv_heads, self.head_dim))
|
||||
|
||||
v = v.unflatten(-1, (self.num_kv_heads, self.head_dim))
|
||||
v = self.v_norm(v)
|
||||
|
||||
attn_out = self.attn(q, k, v, forward_batch=forward_batch)
|
||||
if attn_out.dim() == 3:
|
||||
attn_out = attn_out.flatten(-2, -1)
|
||||
out, _ = self.o_proj(attn_out)
|
||||
return out
|
||||
|
||||
|
||||
class EmbeddingGemma2PLEBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.per_layer_input_gate = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size_per_layer_input,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("per_layer_input_gate", prefix),
|
||||
)
|
||||
self.per_layer_projection = ReplicatedLinear(
|
||||
config.hidden_size_per_layer_input,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("per_layer_projection", prefix),
|
||||
)
|
||||
self.post_per_layer_input_norm = EmbeddingGemma2RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.act_fn = get_act_fn(config.hidden_activation)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, per_layer_input: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
gate = self.act_fn(self.per_layer_input_gate(hidden_states)[0])
|
||||
gated = gate * per_layer_input
|
||||
proj = self.per_layer_projection(gated)[0]
|
||||
return hidden_states + self.post_per_layer_input_norm(proj)
|
||||
|
||||
|
||||
class EmbeddingGemma2DecoderLayer(nn.Module):
|
||||
"""Decoder layer for EmbeddingGemma2.
|
||||
|
||||
Combines self-attention (with sliding or full attention depending on layer index),
|
||||
feed-forward MLP, and Per-Layer Embedding (PLE) block, followed by scaling by a
|
||||
persistent scalar parameter (`layer_scalar`).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer_idx: int,
|
||||
config: PretrainedConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.self_attn = EmbeddingGemma2Attention(
|
||||
config, layer_idx, quant_config, prefix=add_prefix("self_attn", prefix)
|
||||
)
|
||||
self.mlp = Gemma4MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_activation=config.hidden_activation,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
self.ple_block = EmbeddingGemma2PLEBlock(
|
||||
config, quant_config=quant_config, prefix=add_prefix("ple_block", prefix)
|
||||
)
|
||||
self.input_layernorm = EmbeddingGemma2RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.post_attention_layernorm = EmbeddingGemma2RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.pre_feedforward_layernorm = EmbeddingGemma2RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.post_feedforward_layernorm = EmbeddingGemma2RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.register_buffer("layer_scalar", torch.ones(1), persistent=True)
|
||||
self.layer_scalar: torch.Tensor
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
h: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
per_layer_input: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
r = h
|
||||
h = (
|
||||
self.post_attention_layernorm(
|
||||
self.self_attn(
|
||||
positions, self.input_layernorm(h), forward_batch=forward_batch
|
||||
)
|
||||
)
|
||||
+ r
|
||||
)
|
||||
r = h
|
||||
h = (
|
||||
self.post_feedforward_layernorm(self.mlp(self.pre_feedforward_layernorm(h)))
|
||||
+ r
|
||||
)
|
||||
h = self.ple_block(h, per_layer_input)
|
||||
return h * self.layer_scalar
|
||||
|
||||
|
||||
class EmbeddingGemma2TextPLE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.num_layers = config.num_hidden_layers
|
||||
self.hidden_dim = config.hidden_size_per_layer_input
|
||||
self.per_layer_model_projection = ColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
self.num_layers * self.hidden_dim,
|
||||
bias=False,
|
||||
gather_output=True,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("per_layer_model_projection", prefix),
|
||||
)
|
||||
self.per_layer_model_projection_scale = config.hidden_size**-0.5
|
||||
self.per_layer_projection_norm = EmbeddingGemma2RMSNorm(
|
||||
self.hidden_dim, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
def forward(self, inputs_embeds: torch.Tensor) -> torch.Tensor:
|
||||
proj, _ = self.per_layer_model_projection(inputs_embeds)
|
||||
proj = proj * self.per_layer_model_projection_scale
|
||||
proj = proj.reshape(*inputs_embeds.shape[:-1], self.num_layers, self.hidden_dim)
|
||||
return self.per_layer_projection_norm(proj)
|
||||
|
||||
|
||||
class EmbeddingGemma2VocabParallelEmbedding(VocabParallelEmbedding):
|
||||
"""Embedding that replaces multimodal placeholder tokens with pad_token_id before lookup."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
pad_token_id: int = 0,
|
||||
mm_token_ids: tuple[int, ...] = (),
|
||||
embed_scale: float = 1.0,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.pad_token_id = pad_token_id
|
||||
self.mm_token_ids = mm_token_ids
|
||||
self.embed_scale_val = embed_scale
|
||||
self.register_buffer("embed_scale", torch.tensor(embed_scale), persistent=False)
|
||||
self.embed_scale: torch.Tensor
|
||||
|
||||
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
if self.mm_token_ids:
|
||||
for mm_id in self.mm_token_ids:
|
||||
if (input_ids == mm_id).any():
|
||||
input_ids = torch.where(
|
||||
input_ids == mm_id, self.pad_token_id, input_ids
|
||||
)
|
||||
return super().forward(input_ids) * self.embed_scale.to(
|
||||
cast(torch.dtype, self.weight.dtype)
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingGemma2TextModel(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
mm_ids = tuple(
|
||||
tid
|
||||
for tid in (
|
||||
getattr(config, "image_token_id", 258880),
|
||||
getattr(config, "audio_token_id", 258881),
|
||||
getattr(config, "video_token_id", 258884),
|
||||
)
|
||||
if tid is not None
|
||||
)
|
||||
self.embed_tokens = EmbeddingGemma2VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
pad_token_id=config.pad_token_id,
|
||||
mm_token_ids=mm_ids,
|
||||
embed_scale=config.hidden_size**0.5,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("embed_tokens", prefix),
|
||||
)
|
||||
self.ple = EmbeddingGemma2TextPLE(
|
||||
config, quant_config=quant_config, prefix=add_prefix("ple", prefix)
|
||||
)
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
EmbeddingGemma2DecoderLayer(
|
||||
i,
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{i}", prefix),
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
self.norm = EmbeddingGemma2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.embedding_projection = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
getattr(config, "embedding_dim", 768),
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("embedding_projection", prefix),
|
||||
)
|
||||
|
||||
def get_input_embeddings(self) -> nn.Module:
|
||||
return self.embed_tokens
|
||||
|
||||
def dtype(self) -> torch.dtype:
|
||||
return cast(torch.dtype, self.embed_tokens.weight.dtype)
|
||||
|
||||
@property
|
||||
def device(self) -> torch.device:
|
||||
return cast(torch.device, self.embed_tokens.weight.device)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
forward_batch: ForwardBatch | None = None,
|
||||
input_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
if positions is None and forward_batch is not None:
|
||||
positions = getattr(forward_batch, "positions", None)
|
||||
if positions is None:
|
||||
positions = kwargs.get("positions")
|
||||
if input_embeds is None:
|
||||
input_embeds = self.embed_tokens(input_ids)
|
||||
per_layer = self.ple(input_embeds)
|
||||
h = input_embeds
|
||||
for i, layer in enumerate(self.layers):
|
||||
h = layer(positions, h, forward_batch, per_layer[..., i, :])
|
||||
h = self.norm(h)
|
||||
h, _ = self.embedding_projection(h)
|
||||
return h
|
||||
|
||||
|
||||
class EmbeddingGemma2VisionPooler(nn.Module):
|
||||
"""Vision pooler for EmbeddingGemma2.
|
||||
|
||||
Isolated from Gemma4VisionPooler to prevent regressions in generative vision pipelines.
|
||||
Performs 2D spatial average pooling over vision patch tokens into soft visual tokens
|
||||
scaled by root_hidden_size (hidden_size ** 0.5) according to output_length.
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.root_hidden_size = self.hidden_size**0.5
|
||||
|
||||
def _avg_pool_by_positions(
|
||||
self, x: torch.Tensor, patch_positions: torch.Tensor, length: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
input_seq_len = x.shape[1]
|
||||
k = int((input_seq_len // length) ** 0.5)
|
||||
k_squared = k**2
|
||||
if k_squared * length != input_seq_len:
|
||||
raise ValueError(
|
||||
f"Cannot pool {x.shape} to {length}: {k=}^2 times {length=} must be {input_seq_len}."
|
||||
)
|
||||
clamped_positions = patch_positions.clamp(min=0)
|
||||
max_x = clamped_positions[..., 0].max(dim=-1, keepdim=True)[0] + 1
|
||||
kernel_idxs = torch.div(clamped_positions, k, rounding_mode="floor")
|
||||
kernel_idxs = kernel_idxs[..., 0] + (max_x // k) * kernel_idxs[..., 1]
|
||||
|
||||
weights = F.one_hot(kernel_idxs.long(), length).float() / k_squared
|
||||
output = weights.transpose(1, 2) @ x.float()
|
||||
mask = torch.logical_not((weights == 0).all(dim=1))
|
||||
return output, mask
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
patch_positions: torch.Tensor,
|
||||
padding_positions: torch.Tensor,
|
||||
output_length: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if output_length is None:
|
||||
raise ValueError(
|
||||
"output_length is required for EmbeddingGemma2VisionPooler"
|
||||
)
|
||||
if output_length > hidden_states.shape[1]:
|
||||
raise ValueError(
|
||||
f"Cannot output more soft tokens (requested {output_length}) than there are patches"
|
||||
f" ({hidden_states.shape[1]}). Change the value of `num_soft_tokens` when processing."
|
||||
)
|
||||
length = output_length
|
||||
if isinstance(length, (list, tuple)):
|
||||
length = length[0]
|
||||
orig_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.masked_fill(padding_positions.unsqueeze(-1), 0.0)
|
||||
if hidden_states.shape[1] == length:
|
||||
pooler_mask = ~padding_positions
|
||||
else:
|
||||
hidden_states, pooler_mask = self._avg_pool_by_positions(
|
||||
hidden_states, patch_positions, length
|
||||
)
|
||||
hidden_states = (hidden_states.float() * self.root_hidden_size).to(orig_dtype)
|
||||
return hidden_states, pooler_mask
|
||||
|
||||
|
||||
class EmbeddingGemma2Model(nn.Module):
|
||||
"""EmbeddingGemma2 multimodal embedding model.
|
||||
|
||||
Supports text, image, audio, and video inputs with mean pooling and L2 normalization
|
||||
over a 768-dim output embedding space.
|
||||
|
||||
Architecture and design choices:
|
||||
- Selective modality loading: vision and audio towers are conditionally loaded
|
||||
based on server_args.limit_mm_data_per_request to save VRAM when only text,
|
||||
image, audio, or video modalities are served.
|
||||
- EmbeddingGemma2RMSNorm / forward_native rebinding: rebinds forward to forward_native
|
||||
across Gemma4RMSNorm and RMSNorm modules to handle 3D/4D tensors and with_scale=False.
|
||||
- EmbeddingGemma2VisionPooler: dedicated 2D spatial average pooler isolated from
|
||||
generative Gemma4VisionPooler to prevent regressions in generation paths.
|
||||
- Bidirectional sliding window: attention uses inclusive <= 512 bidirectional SWA
|
||||
enabling full bidirectional cross-token representations for embedding tasks.
|
||||
- Non-meta buffer materialization: materializes persistent buffers such as embed_scale
|
||||
while preserving non-meta buffers (including scalar/tensor buffers with inf/-inf).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
text_config = config.text_config
|
||||
if hasattr(text_config, "allow_global_per_layer_attribute_access"):
|
||||
text_config.allow_global_per_layer_attribute_access = True
|
||||
self.text_config = text_config
|
||||
|
||||
try:
|
||||
limit_mm = get_mm().limit_mm_data_per_request or {}
|
||||
except Exception: # noqa: BLE001
|
||||
limit_mm = {}
|
||||
|
||||
# Gated Vision Tower
|
||||
has_vision = getattr(config, "vision_config", None) is not None and (
|
||||
limit_mm.get("image", 1) > 0 or limit_mm.get("video", 1) > 0
|
||||
)
|
||||
self.vision_tower: Gemma4VisionEncoder | None
|
||||
self.embed_vision: Gemma4MultimodalEmbedder | None
|
||||
self.audio_tower: Gemma4AudioEncoder | None
|
||||
self.embed_audio: Gemma4MultimodalEmbedder | None
|
||||
|
||||
if has_vision:
|
||||
self.vision_tower = Gemma4VisionEncoder(
|
||||
config.vision_config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("vision_tower", prefix),
|
||||
)
|
||||
self.vision_tower.pooler = EmbeddingGemma2VisionPooler(config.vision_config) # type: ignore[assignment]
|
||||
for layer in self.vision_tower.encoder.layers: # type: ignore[union-attr]
|
||||
layer.self_attn.qkv_backend = VisionSdpaAttention( # type: ignore[attr-defined,union-attr]
|
||||
head_dim=config.vision_config.head_dim,
|
||||
num_heads=layer.self_attn.num_heads_per_partition, # type: ignore[attr-defined,arg-type,union-attr]
|
||||
num_kv_heads=layer.self_attn.num_kv_heads_per_partition, # type: ignore[attr-defined,arg-type,union-attr]
|
||||
dropout=0.0,
|
||||
flatten_batch=False,
|
||||
softmax_in_single_precision=False,
|
||||
softmax_scale=1.0,
|
||||
)
|
||||
self.embed_vision = Gemma4MultimodalEmbedder(
|
||||
config.vision_config,
|
||||
text_config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("embed_vision", prefix),
|
||||
)
|
||||
else:
|
||||
self.vision_tower = None
|
||||
self.embed_vision = None
|
||||
|
||||
# Gated Audio Tower
|
||||
has_audio = (
|
||||
getattr(config, "audio_config", None) is not None
|
||||
and limit_mm.get("audio", 1) > 0
|
||||
)
|
||||
if has_audio:
|
||||
self.audio_tower = Gemma4AudioEncoder(
|
||||
config.audio_config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("audio_tower", prefix),
|
||||
)
|
||||
self.embed_audio = Gemma4MultimodalEmbedder(
|
||||
config.audio_config,
|
||||
text_config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("embed_audio", prefix),
|
||||
)
|
||||
else:
|
||||
self.audio_tower = None
|
||||
self.embed_audio = None
|
||||
|
||||
self.language_model = EmbeddingGemma2TextModel(
|
||||
text_config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("language_model", prefix),
|
||||
)
|
||||
self.pooler = Pooler(PoolingType.MEAN, normalize=True)
|
||||
|
||||
for m in self.modules():
|
||||
if isinstance(m, (Gemma4RMSNorm, RMSNorm)):
|
||||
m.forward = m.forward_native # type: ignore[method-assign,assignment]
|
||||
|
||||
def get_attention_sliding_window_size(self) -> int:
|
||||
return 512
|
||||
|
||||
def get_input_embeddings(self) -> nn.Module:
|
||||
return self.language_model.get_input_embeddings()
|
||||
|
||||
def pad_input_ids(
|
||||
self, input_ids: list[int], mm_inputs: MultimodalInputs
|
||||
) -> list[int]:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
|
||||
if self.vision_tower is None or self.embed_vision is None:
|
||||
raise ValueError(
|
||||
"Image inputs provided but vision tower is not loaded. "
|
||||
"Ensure limit_mm_data_per_request allows image > 0."
|
||||
)
|
||||
vt = self.vision_tower
|
||||
all_embeds = []
|
||||
for item in items:
|
||||
all_pixel_values = flatten_nested_list([item.feature])
|
||||
all_position_ids = flatten_nested_list(
|
||||
[getattr(item, "image_position_ids", None)]
|
||||
)
|
||||
for pv_idx, pv in enumerate(all_pixel_values):
|
||||
if pv.dim() in (2, 3) and pv.shape[-1] == self.text_config.hidden_size:
|
||||
all_embeds.append(pv.to(self.language_model.device))
|
||||
continue
|
||||
if pv_idx >= len(all_position_ids) or all_position_ids[pv_idx] is None:
|
||||
raise ValueError(
|
||||
f"pixel_values[{pv_idx}] has no matching image_position_ids."
|
||||
)
|
||||
pp = all_position_ids[pv_idx]
|
||||
if pv.dim() == 2:
|
||||
pv = pv.unsqueeze(0)
|
||||
if pp.dim() == 2:
|
||||
pp = pp.unsqueeze(0)
|
||||
pv = pv.to(device=vt.device, dtype=self.language_model.dtype())
|
||||
pp = pp.to(device=vt.device)
|
||||
|
||||
pooled, pooler_mask = vt(pv, pp)
|
||||
for hs, mask in zip(pooled, pooler_mask):
|
||||
real_tokens = hs[mask]
|
||||
all_embeds.append(
|
||||
self.embed_vision(
|
||||
inputs_embeds=real_tokens.unsqueeze(0)
|
||||
).squeeze(0)
|
||||
)
|
||||
if all_embeds:
|
||||
return torch.cat(all_embeds, dim=0)
|
||||
else:
|
||||
return torch.empty(
|
||||
0,
|
||||
self.text_config.hidden_size,
|
||||
device=next(self.parameters()).device,
|
||||
dtype=self.language_model.dtype(),
|
||||
)
|
||||
|
||||
def get_video_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
|
||||
if self.vision_tower is None or self.embed_vision is None:
|
||||
raise ValueError(
|
||||
"Video inputs provided but vision tower is not loaded. "
|
||||
"Ensure limit_mm_data_per_request allows video > 0."
|
||||
)
|
||||
vt = self.vision_tower
|
||||
all_embeds = []
|
||||
for item in items:
|
||||
all_pixel_values = flatten_nested_list([item.feature])
|
||||
all_position_ids = flatten_nested_list(
|
||||
[getattr(item, "video_position_ids", None)]
|
||||
)
|
||||
for pv_idx, pv in enumerate(all_pixel_values):
|
||||
if pv.dim() in (2, 3) and pv.shape[-1] == self.text_config.hidden_size:
|
||||
all_embeds.append(pv.to(self.language_model.device))
|
||||
continue
|
||||
if pv_idx >= len(all_position_ids) or all_position_ids[pv_idx] is None:
|
||||
raise ValueError(
|
||||
f"pixel_values_videos[{pv_idx}] has no matching video_position_ids."
|
||||
)
|
||||
pp = all_position_ids[pv_idx]
|
||||
if pv.dim() == 4:
|
||||
pv = pv.reshape(-1, pv.shape[-2], pv.shape[-1])
|
||||
if pp.dim() == 4:
|
||||
pp = pp.reshape(-1, pp.shape[-2], pp.shape[-1])
|
||||
if pv.dim() == 2:
|
||||
pv = pv.unsqueeze(0)
|
||||
if pp.dim() == 2:
|
||||
pp = pp.unsqueeze(0)
|
||||
pv = pv.to(device=vt.device, dtype=self.language_model.dtype())
|
||||
pp = pp.to(device=vt.device)
|
||||
|
||||
pooled, pooler_mask = vt(pv, pp)
|
||||
for hs, mask in zip(pooled, pooler_mask):
|
||||
real_tokens = hs[mask]
|
||||
all_embeds.append(
|
||||
self.embed_vision(
|
||||
inputs_embeds=real_tokens.unsqueeze(0)
|
||||
).squeeze(0)
|
||||
)
|
||||
if all_embeds:
|
||||
return torch.cat(all_embeds, dim=0)
|
||||
else:
|
||||
return torch.empty(
|
||||
0,
|
||||
self.text_config.hidden_size,
|
||||
device=next(self.parameters()).device,
|
||||
dtype=self.language_model.dtype(),
|
||||
)
|
||||
|
||||
def get_audio_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
|
||||
if self.audio_tower is None or self.embed_audio is None:
|
||||
raise ValueError(
|
||||
"Audio inputs provided but audio tower is not loaded. "
|
||||
"Ensure limit_mm_data_per_request allows audio > 0."
|
||||
)
|
||||
all_input_features = flatten_nested_list([item.feature for item in items])
|
||||
all_input_features_mask = flatten_nested_list(
|
||||
[~item.input_features_mask for item in items] # type: ignore[operator]
|
||||
)
|
||||
all_embeds = []
|
||||
for input_features, input_features_mask in zip(
|
||||
all_input_features, all_input_features_mask
|
||||
):
|
||||
if input_features.dim() == 2:
|
||||
input_features = input_features.unsqueeze(0)
|
||||
if input_features_mask.dim() == 1:
|
||||
input_features_mask = input_features_mask.unsqueeze(0)
|
||||
|
||||
input_features = input_features.to(
|
||||
device=self.audio_tower.device,
|
||||
dtype=self.language_model.dtype(),
|
||||
)
|
||||
input_features_mask = input_features_mask.to(device=input_features.device)
|
||||
|
||||
audio_encodings, audio_mask = self.audio_tower(
|
||||
input_features, input_features_mask
|
||||
)
|
||||
audio_features = self.embed_audio(inputs_embeds=audio_encodings)
|
||||
for enc, mask in zip(audio_features, audio_mask):
|
||||
all_embeds.append(enc[~mask])
|
||||
if all_embeds:
|
||||
return torch.cat(all_embeds, dim=0)
|
||||
else:
|
||||
return torch.empty(
|
||||
0,
|
||||
self.text_config.hidden_size,
|
||||
device=next(self.parameters()).device,
|
||||
dtype=self.language_model.dtype(),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: torch.Tensor | None = None,
|
||||
get_embedding: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
data_embedding_funcs: dict[
|
||||
Modality, Callable[[list[MultimodalDataItem]], Any]
|
||||
] = {}
|
||||
if self.vision_tower is not None:
|
||||
data_embedding_funcs[Modality.IMAGE] = self.get_image_feature
|
||||
data_embedding_funcs[Modality.VIDEO] = self.get_video_feature
|
||||
if self.audio_tower is not None:
|
||||
data_embedding_funcs[Modality.AUDIO] = self.get_audio_feature
|
||||
|
||||
if forward_batch.contains_mm_inputs():
|
||||
h = general_mm_embed_routine(
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
language_model=self.language_model,
|
||||
data_embedding_funcs=data_embedding_funcs,
|
||||
placeholder_tokens={
|
||||
Modality.IMAGE: [getattr(self.config, "image_token_id", 258880)],
|
||||
Modality.AUDIO: [getattr(self.config, "audio_token_id", 258881)],
|
||||
Modality.VIDEO: [getattr(self.config, "video_token_id", 258884)],
|
||||
},
|
||||
positions=positions,
|
||||
)
|
||||
else:
|
||||
h = self.language_model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
input_embeds=input_embeds,
|
||||
)
|
||||
|
||||
# Pooler expects fp32 tensor, mean pool, L2 normalize
|
||||
return self.pooler(h.float(), forward_batch)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> list[str]:
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
buffers_dict = dict(self.named_buffers())
|
||||
loaded_params = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# Tower skipping
|
||||
if self.audio_tower is None and (
|
||||
name.startswith(("audio_tower.", "embed_audio."))
|
||||
):
|
||||
continue
|
||||
if self.vision_tower is None and (
|
||||
name.startswith(("vision_tower.", "embed_vision."))
|
||||
):
|
||||
continue
|
||||
|
||||
# Tower remapping using Gemma4 static methods
|
||||
if name.startswith("audio_tower."):
|
||||
name = Gemma4ForConditionalGeneration._remap_audio_tower_name(name)
|
||||
if name.startswith(
|
||||
("vision_tower.", "audio_tower.", "embed_vision.", "embed_audio.")
|
||||
):
|
||||
name = Gemma4ForConditionalGeneration._remap_tower_name(
|
||||
name, params_dict
|
||||
)
|
||||
|
||||
# Persistent buffer loading (e.g. layer_scalar)
|
||||
if name in buffers_dict:
|
||||
buffers_dict[name].copy_(loaded_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
# Stacked projection routing
|
||||
for param_name, shard_name, shard_id in stacked_params_mapping:
|
||||
if shard_name in name:
|
||||
mapped_name = name.replace(shard_name, param_name)
|
||||
if mapped_name in params_dict:
|
||||
param = params_dict[mapped_name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
try:
|
||||
weight_loader(param, loaded_weight, shard_id) # type: ignore[call-arg]
|
||||
except TypeError:
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(mapped_name)
|
||||
break
|
||||
else:
|
||||
if name in params_dict:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
# Audit unloaded parameters
|
||||
expected_params = set(params_dict.keys())
|
||||
if self.vision_tower is None:
|
||||
expected_params = {
|
||||
k
|
||||
for k in expected_params
|
||||
if not k.startswith(("vision_tower.", "embed_vision."))
|
||||
}
|
||||
if self.audio_tower is None:
|
||||
expected_params = {
|
||||
k
|
||||
for k in expected_params
|
||||
if not k.startswith(("audio_tower.", "embed_audio."))
|
||||
}
|
||||
unloaded = expected_params - loaded_params
|
||||
if unloaded:
|
||||
logger.warning(
|
||||
f"EmbeddingGemma2Model: {len(unloaded)} expected parameters were not loaded: {sorted(unloaded)[:10]}"
|
||||
)
|
||||
|
||||
# Materialize non-persistent / meta buffers
|
||||
target_device = self.language_model.device
|
||||
target_dtype = self.language_model.dtype()
|
||||
for module in self.modules():
|
||||
for name, buf in list(module.named_buffers(recurse=False)):
|
||||
if buf is not None and (buf.is_meta or buf.device != target_device):
|
||||
if name == "embed_scale":
|
||||
scale_val = getattr(
|
||||
module,
|
||||
"embed_scale_val",
|
||||
getattr(self.config, "hidden_size", 512) ** 0.5,
|
||||
)
|
||||
module._buffers[name] = torch.tensor(
|
||||
scale_val, dtype=target_dtype, device=target_device
|
||||
)
|
||||
elif not buf.is_meta:
|
||||
module._buffers[name] = buf.to(
|
||||
device=target_device, dtype=target_dtype
|
||||
)
|
||||
else:
|
||||
module._buffers[name] = torch.ones(
|
||||
buf.shape, dtype=target_dtype, device=target_device
|
||||
)
|
||||
|
||||
for m in self.modules():
|
||||
if isinstance(m, (Gemma4RMSNorm, RMSNorm)):
|
||||
m.forward = m.forward_native # type: ignore[method-assign,assignment]
|
||||
|
||||
return list(loaded_params)
|
||||
|
||||
def get_weights_by_name(
|
||||
self, name: str, truncate_size: int = 100, tp_size: int = 1
|
||||
) -> torch.Tensor | None:
|
||||
params_dict = dict(self.named_parameters())
|
||||
if name in params_dict:
|
||||
return (
|
||||
params_dict[name]
|
||||
.data.cpu()
|
||||
.to(torch.float32)
|
||||
.numpy()
|
||||
.tolist()[:truncate_size]
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
EntryClass = [EmbeddingGemma2Model]
|
||||
@@ -0,0 +1,276 @@
|
||||
# Copyright 2025 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
"""Multimodal processor for EmbeddingGemma2Model (EmbeddingGemma v2)."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from sglang.srt.models.embedding_gemma2 import EmbeddingGemma2Model
|
||||
from sglang.srt.multimodal.processors.base_processor import (
|
||||
Modality,
|
||||
MultimodalSpecialTokens,
|
||||
)
|
||||
from sglang.srt.multimodal.processors.gemma4 import Gemma4SGLangProcessor
|
||||
from sglang.srt.utils.common import load_audio
|
||||
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _load_audio_from_memory(
|
||||
audio_file: Any, sr: int | None = 16000, mono: bool = True
|
||||
) -> np.ndarray:
|
||||
"""Load or resample audio waveforms from in-memory numpy array, torch tensor, tuple, or path."""
|
||||
if sr is None:
|
||||
sr = 16000
|
||||
|
||||
input_sr = None
|
||||
if isinstance(audio_file, tuple) and len(audio_file) == 2:
|
||||
audio_file, input_sr = audio_file
|
||||
|
||||
if isinstance(audio_file, torch.Tensor):
|
||||
audio_file = audio_file.detach().cpu().numpy()
|
||||
|
||||
if isinstance(audio_file, np.ndarray):
|
||||
waveform = audio_file.astype(np.float32, copy=False)
|
||||
if waveform.ndim > 1 and mono:
|
||||
if waveform.shape[0] <= 8 and waveform.shape[0] < waveform.shape[-1]:
|
||||
waveform = waveform.mean(axis=0)
|
||||
else:
|
||||
waveform = waveform.mean(axis=-1)
|
||||
if (
|
||||
input_sr is not None
|
||||
and sr is not None
|
||||
and input_sr != sr
|
||||
and len(waveform) > 0
|
||||
):
|
||||
try:
|
||||
import torchaudio # type: ignore[import-untyped]
|
||||
|
||||
audio_tensor = torch.from_numpy(waveform).float()
|
||||
resampler = torchaudio.transforms.Resample(
|
||||
orig_freq=input_sr, new_freq=sr
|
||||
)
|
||||
waveform = resampler(audio_tensor).numpy().astype(np.float32)
|
||||
except Exception: # noqa: BLE001
|
||||
from scipy import signal # type: ignore[import-untyped]
|
||||
|
||||
num_samples = int(len(waveform) * float(sr) / input_sr)
|
||||
waveform = signal.resample(waveform, num_samples).astype(np.float32)
|
||||
return waveform
|
||||
|
||||
return load_audio(audio_file, sr=sr, mono=mono)
|
||||
|
||||
|
||||
class EmbeddingGemma2SGLangProcessor(Gemma4SGLangProcessor):
|
||||
"""Multimodal input processor for EmbeddingGemma2Model.
|
||||
|
||||
Preprocesses image, video, and audio inputs for EmbeddingGemma v2:
|
||||
- Prompt retention: retains prompt task/title prefixes verbatim, disabling the
|
||||
anchored prompt suppression used in generative Gemma4.
|
||||
- Media-only requests: synthesizes the placeholder prompt the HF processor builds
|
||||
when no text is given, and rejects placeholder/media count mismatches.
|
||||
- Video processing: samples frame indices with the HF video processor's
|
||||
fps/max_frames rule before decoding, so only sampled frames are materialized.
|
||||
- Audio processing: provides unpadded raw waveforms (16kHz mono float32).
|
||||
"""
|
||||
|
||||
models: list[Any] = [EmbeddingGemma2Model] # type: ignore[assignment] # noqa: RUF012
|
||||
|
||||
# Plain nvJPEG skips fancy chroma upsampling and drifts from the PIL decode the
|
||||
# checkpoint was validated with (embedding cos ~0.99 vs HF); nvjpeg_fancy matches it.
|
||||
gpu_image_decode = "nvjpeg_fancy"
|
||||
|
||||
def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
|
||||
super().__init__(hf_config, server_args, _processor, *args, **kwargs)
|
||||
self.disable_fast_image_processor = True
|
||||
|
||||
# Resolve special tokens
|
||||
self.IM_START_TOKEN_ID = getattr(hf_config, "boi_token_id", 255999)
|
||||
self.IM_END_TOKEN_ID = getattr(hf_config, "eoi_token_id", 258882)
|
||||
self.IMAGE_TOKEN_ID = getattr(hf_config, "image_token_id", 258880)
|
||||
self.AUDIO_TOKEN_ID = getattr(hf_config, "audio_token_id", 258881)
|
||||
self.VIDEO_TOKEN_ID = getattr(hf_config, "video_token_id", 258884)
|
||||
self.AUDIO_START_TOKEN_ID = getattr(hf_config, "boa_token_id", 256000)
|
||||
self.AUDIO_END_TOKEN_ID = getattr(
|
||||
hf_config, "eoa_token_id", getattr(hf_config, "eoa_token_index", 258883)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _load_single_item(
|
||||
cls,
|
||||
data: Any,
|
||||
modality: Modality,
|
||||
frame_count_limit: int | None = None,
|
||||
audio_sample_rate: int | None = None,
|
||||
discard_alpha_channel: bool = True,
|
||||
) -> Any:
|
||||
if modality == Modality.AUDIO:
|
||||
return _load_audio_from_memory(data, sr=audio_sample_rate, mono=True)
|
||||
return super()._load_single_item(
|
||||
data,
|
||||
modality=modality,
|
||||
frame_count_limit=frame_count_limit,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
discard_alpha_channel=discard_alpha_channel,
|
||||
)
|
||||
|
||||
async def load_mm_data( # type: ignore[override]
|
||||
self,
|
||||
prompt: Any,
|
||||
multimodal_tokens: MultimodalSpecialTokens,
|
||||
image_data: list[Any] | None = None,
|
||||
video_data: list[Any] | None = None,
|
||||
audio_data: list[Any] | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
# Resolve the prompt here rather than overriding process_mm_data_async, so
|
||||
# requests keep Gemma4's route through the preprocessing worker pool.
|
||||
prompt = self._resolve_prompt(
|
||||
prompt,
|
||||
n_image=len(image_data) if image_data else 0,
|
||||
n_video=len(video_data) if video_data else 0,
|
||||
n_audio=len(audio_data) if audio_data else 0,
|
||||
)
|
||||
return await super().load_mm_data(
|
||||
prompt, multimodal_tokens, image_data, video_data, audio_data, **kwargs
|
||||
)
|
||||
|
||||
def _resolve_prompt(
|
||||
self, input_text: Any, *, n_image: int, n_video: int, n_audio: int
|
||||
) -> Any:
|
||||
if not input_text:
|
||||
# Same layout HF builds for text=None: images, then videos, then audio.
|
||||
tokens = self.mm_tokens
|
||||
return " ".join(
|
||||
[tokens.image_token] * n_image
|
||||
+ [tokens.video_token] * n_video
|
||||
+ [tokens.audio_token] * n_audio
|
||||
)
|
||||
prompt = (
|
||||
self._tokenizer.decode(input_text)
|
||||
if isinstance(input_text, list)
|
||||
else input_text
|
||||
)
|
||||
found = {Modality.IMAGE: 0, Modality.VIDEO: 0, Modality.AUDIO: 0}
|
||||
for part in re.split(self.mm_tokens.get_combined_regex(), prompt):
|
||||
modality = self.mm_tokens.get_modality_of_token(part)
|
||||
if modality is not None:
|
||||
found[modality] += 1
|
||||
expected = {
|
||||
Modality.IMAGE: n_image,
|
||||
Modality.VIDEO: n_video,
|
||||
Modality.AUDIO: n_audio,
|
||||
}
|
||||
# The generic loader silently drops unmatched media, which yields a wrong
|
||||
# embedding instead of an error; HF raises here too.
|
||||
mismatched = [
|
||||
f"{m.name.lower()}: {found[m]} placeholder(s) vs {expected[m]} item(s)"
|
||||
for m in expected
|
||||
if found[m] != expected[m]
|
||||
]
|
||||
if mismatched:
|
||||
raise ValueError(
|
||||
"EmbeddingGemma2 needs exactly one <|image|>/<|video|>/<|audio|> "
|
||||
"placeholder per media item, or no text at all; got "
|
||||
+ ", ".join(mismatched)
|
||||
+ "."
|
||||
)
|
||||
return input_text
|
||||
|
||||
def _sample_video(self, video: Any) -> tuple[torch.Tensor, VideoMetadata]:
|
||||
"""Decode only the frames the HF video processor would keep."""
|
||||
if isinstance(video, VideoDecoderWrapper):
|
||||
total, src_fps = len(video), video.avg_fps
|
||||
else:
|
||||
frames, meta = video if isinstance(video, tuple) else (video, None)
|
||||
if isinstance(meta, VideoMetadata):
|
||||
src_fps = meta.fps
|
||||
elif isinstance(meta, dict):
|
||||
src_fps = meta.get("fps")
|
||||
else:
|
||||
src_fps = None
|
||||
total = len(frames)
|
||||
metadata = VideoMetadata(
|
||||
total_num_frames=total,
|
||||
fps=src_fps,
|
||||
duration=total / src_fps if src_fps else None,
|
||||
)
|
||||
video_processor = self._processor.video_processor
|
||||
indices = video_processor.sample_frames(
|
||||
metadata,
|
||||
fps=self.video_config.get("fps", video_processor.fps),
|
||||
max_frames=self.video_config.get("max_frames", video_processor.max_frames),
|
||||
overflow_strategy=self.video_config.get(
|
||||
"overflow_strategy", video_processor.overflow_strategy
|
||||
),
|
||||
).tolist()
|
||||
if isinstance(video, VideoDecoderWrapper):
|
||||
sampled = torch.as_tensor(video.get_frames_at(indices)).permute(0, 3, 1, 2)
|
||||
elif isinstance(frames, torch.Tensor):
|
||||
sampled = frames[indices]
|
||||
else:
|
||||
sampled = torch.as_tensor(np.asarray(frames)[indices])
|
||||
metadata.frames_indices = indices
|
||||
return sampled.contiguous(), metadata
|
||||
|
||||
def process_mm_data( # type: ignore[override]
|
||||
self,
|
||||
input_text: Any,
|
||||
images: list[Any] | None = None,
|
||||
videos: list[Any] | None = None,
|
||||
audios: list[Any] | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
if "num_frames" in kwargs or "num_frames" in kwargs.get("videos_kwargs", {}):
|
||||
raise ValueError(
|
||||
"Sampling with `num_frames` is not supported for EmbeddingGemma2. "
|
||||
"Use `fps` and `max_frames` instead."
|
||||
)
|
||||
|
||||
# 1. Prompt retention: do NOT suppress task/title prefixes.
|
||||
# input_text passes through verbatim.
|
||||
|
||||
# 2. Audio: unpadded raw waveforms (16kHz mono float32)
|
||||
if audios:
|
||||
loaded_audios = []
|
||||
for a in audios:
|
||||
if isinstance(a, (np.ndarray, torch.Tensor, tuple)):
|
||||
loaded_audios.append(
|
||||
_load_audio_from_memory(a, sr=16000, mono=True)
|
||||
)
|
||||
else:
|
||||
loaded_audios.append(np.asarray(a, dtype=np.float32))
|
||||
kwargs["audio"] = loaded_audios
|
||||
kwargs.setdefault("audio_kwargs", {})["truncation"] = False
|
||||
audios = None
|
||||
|
||||
# 3. Video: frames are sampled here, so the HF processor must not resample.
|
||||
if videos:
|
||||
sampled = [self._sample_video(v) for v in videos]
|
||||
videos = [frames for frames, _ in sampled]
|
||||
kwargs.setdefault("videos_kwargs", {})["video_metadata"] = [
|
||||
meta for _, meta in sampled
|
||||
]
|
||||
kwargs["do_sample_frames"] = False
|
||||
|
||||
return super(Gemma4SGLangProcessor, self).process_mm_data(
|
||||
input_text, images=images, videos=videos, audios=audios, **kwargs
|
||||
)
|
||||
@@ -64,6 +64,7 @@ _GEMMA4_MODEL_TYPES = (
|
||||
"gemma4_unified",
|
||||
"gemma4_unified_assistant",
|
||||
"diffusion_gemma",
|
||||
"embedding_gemma2",
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user