From d1ba943016cb93fe874b924bbd293214ff086413 Mon Sep 17 00:00:00 2001 From: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:54:45 -0700 Subject: [PATCH] [Model] Add support for EmbeddingGemma 2 (#42782) Signed-off-by: Luciano Martins Co-authored-by: Luciano Martins Co-authored-by: Claude Opus 5.5 --- .../autoregressive/Google/EmbeddingGemma2.mdx | 188 ++++ docs/docs.json | 1 + .../kernels/ops/attention/extend_attention.py | 18 +- python/sglang/srt/arg_groups/model_hook.py | 18 + .../arg_groups/model_overrides/__init__.py | 1 + .../model_overrides/embedding_gemma2.py | 43 + .../srt/configs/embedding_model_spec.py | 34 +- python/sglang/srt/configs/model_config.py | 13 +- python/sglang/srt/managers/io_struct.py | 75 +- .../sglang/srt/managers/tokenizer_manager.py | 6 +- python/sglang/srt/models/embedding_gemma2.py | 963 ++++++++++++++++++ .../multimodal/processors/embedding_gemma2.py | 276 +++++ .../srt/utils/hf_transformers/config.py | 1 + .../unit/configs/test_embedding_model_spec.py | 20 +- .../unit/managers/test_io_struct.py | 33 + 15 files changed, 1669 insertions(+), 21 deletions(-) create mode 100644 docs/cookbook/autoregressive/Google/EmbeddingGemma2.mdx create mode 100644 python/sglang/srt/arg_groups/model_overrides/embedding_gemma2.py create mode 100644 python/sglang/srt/models/embedding_gemma2.py create mode 100644 python/sglang/srt/multimodal/processors/embedding_gemma2.py diff --git a/docs/cookbook/autoregressive/Google/EmbeddingGemma2.mdx b/docs/cookbook/autoregressive/Google/EmbeddingGemma2.mdx new file mode 100644 index 0000000000..8eebfb1916 --- /dev/null +++ b/docs/cookbook/autoregressive/Google/EmbeddingGemma2.mdx @@ -0,0 +1,188 @@ +--- +title: EmbeddingGemma 2 +description: Serve Google's multimodal EmbeddingGemma 2 model (text, image, video, audio) with SGLang. +tag: NEW +--- + +## Overview + +[EmbeddingGemma 2](https://huggingface.co/google/embeddinggemma-2) is Google's multimodal embedding model. It maps text (including code), images, video, audio, and interleaved combinations of them into one shared, L2-normalized 768-dimensional vector space. + +| Property | Value | +| :-- | :-- | +| Parameters | 740M total: 270M text backbone, 170M vision encoder, 300M audio encoder | +| Architecture | 24-layer bidirectional Gemma 4 encoder, 5:1 local:global attention, local window radius 512 | +| Context | 8,192 tokens shared by all modalities in a request | +| Output | 768 dimensions; Matryoshka truncation to 512, 256, or 128 | +| Pooling | Mean pooling over all tokens, then L2 normalization | + +SGLang detects the `EmbeddingGemma2Model` architecture and configures it automatically: + +- enables embedding mode, so `--is-embedding` is not needed; +- selects the Triton attention backend, which implements the bidirectional sliding window; +- disables RadixAttention prefix caching and chunked prefill, because bidirectional attention makes every token depend on the whole input; +- caps the context length at 8,192 tokens. + +## Prerequisites + +- An NVIDIA CUDA GPU. The model weighs about 1.5 GB in BF16, so any data-center GPU works. +- A Transformers build that includes EmbeddingGemma 2 (`model_type: embedding_gemma2`). Older releases fail at startup with `Transformers does not recognize this architecture`. +- An SGLang build that includes EmbeddingGemma 2 support: + +```bash +pip install 'git+https://github.com/sgl-project/sglang.git#subdirectory=python' +``` + +Run the model in BF16 (the checkpoint default) or FP32. Do not use FP16: the model's activations exceed its range and produce NaN or degraded embeddings. + +## Start the server + +```bash +sglang serve \ + --model-path google/embeddinggemma-2 \ + --host 0.0.0.0 +``` + +### Load only the encoders you need + +The vision and audio encoders load only when their modality is allowed. Set a modality's per-request limit to `0` to skip loading its encoder. Images and video share the vision encoder. + +| Serving | `--limit-mm-data-per-request` | Parameters loaded | +| :-- | :-- | :-- | +| Text only | `'{"image": 0, "video": 0, "audio": 0}'` | 270M | +| Text and images/video | `'{"audio": 0}'` | 440M | +| Text and audio | `'{"image": 0, "video": 0}'` | 570M | +| All modalities | (default) | 740M | + +```bash +sglang serve \ + --model-path google/embeddinggemma-2 \ + --limit-mm-data-per-request '{"image": 0, "video": 0, "audio": 0}' \ + --host 0.0.0.0 +``` + +A request for a disabled modality fails with HTTP 400, for example `Image count 1 exceeds limit 0 per request.` + +### Enable Matryoshka dimensions + +To let clients request shorter vectors through the `dimensions` field, declare the supported sizes when starting the server: + +```bash +sglang serve \ + --model-path google/embeddinggemma-2 \ + --json-model-override-args '{"matryoshka_dimensions": [128, 256, 512, 768]}' \ + --host 0.0.0.0 +``` + +SGLang truncates the pooled vector and then re-normalizes it, as the model card requires. Other sizes are rejected with HTTP 400. Queries and documents must use the same dimension. + +## Task prompts + +EmbeddingGemma 2 is trained with short task prefixes on text inputs. Prepend them yourself; images, video, and audio take no prefix. For documents without a title, use `title: none`. + +| Use case | Query | Document | +| :-- | :-- | :-- | +| Retrieval | `task: search result \| query: {content}` | `title: {title} \| text: {content}` | +| Question answering | `task: question answering \| query: {content}` | `title: {title} \| text: {content}` | +| Fact checking | `task: fact checking \| query: {content}` | `title: {title} \| text: {content}` | +| Code retrieval | `task: code retrieval \| query: {content}` | `title: {title} \| text: {content}` | +| Classification | `task: classification \| query: {content}` | | +| Clustering | `task: clustering \| query: {content}` | | +| Semantic similarity | `task: sentence similarity \| query: {content}` | | + +## Create embeddings + +### Native `/encode` API + +`/encode` supports every modality, including several media items in one embedding. + +Text, one embedding per string: + +```bash +curl http://127.0.0.1:30000/encode \ + -H 'Content-Type: application/json' \ + -d '{ + "text": [ + "task: search result | query: What causes the aurora borealis?", + "title: none | text: The aurora is caused by solar wind particles colliding with the upper atmosphere." + ] + }' +``` + +An image, video, or audio clip on its own. SGLang inserts the placeholder for you: + +```bash +curl http://127.0.0.1:30000/encode \ + -H 'Content-Type: application/json' \ + -d '{"image_data": "https://example.com/cat.jpg"}' +``` + +Use `video_data` or `audio_data` the same way. Audio should be mono at 16 kHz. + +Interleaved input, one embedding for the whole sequence. Mark each item's position with `<|image|>`, `<|video|>`, or `<|audio|>`, and pass the items in the same order: + +```bash +curl http://127.0.0.1:30000/encode \ + -H 'Content-Type: application/json' \ + -d '{ + "text": "Waterproof running shoes. <|image|> Breathable mesh upper. <|image|> Narration: <|audio|>", + "image_data": ["https://example.com/shoe.jpg", "https://example.com/mesh.jpg"], + "audio_data": "https://example.com/narration.wav" + }' +``` + +#### How requests map to embeddings + +| Request shape | Result | +| :-- | :-- | +| `text` is a string, media is a list | 1 embedding containing all the media | +| `text` is a list of N strings, media is a list of N entries | N embeddings, paired by position; an entry may be `null` or a list of several items | +| `text` is a list, media is a single item (not a list) | the item is attached to every request, so every string needs its placeholder | +| No `text`, media is a flat list `[a, b]` | 1 embedding of `a` and `b` together | +| No `text`, media is a list of lists `[[a], [b]]` | 2 embeddings | + +Requests are rejected with HTTP 400 instead of being silently changed when: + +- the number of `<|image|>`, `<|video|>`, or `<|audio|>` placeholders differs from the number of items of that modality (text without any placeholder plus an image is also a mismatch); +- a batched media list has a different length than the `text` list. + +### OpenAI-compatible `/v1/embeddings` + +```bash +curl http://127.0.0.1:30000/v1/embeddings \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "google/embeddinggemma-2", + "input": [ + {"text": "a product photo: <|image|>", "image": "https://example.com/shoe.jpg"}, + {"image": "https://example.com/mesh.jpg"}, + {"text": "task: search result | query: waterproof trail shoes"} + ] + }' +``` + +Each input item produces one embedding. In this API, an item carries at most one `image` or `video` and no audio. For several media items in one embedding, or for audio, use `/encode`. + +## Input limits + +All modalities share the 8,192-token context. Requests that exceed it fail with HTTP 400 (`The input (N tokens) is longer than the model's context length (8192 tokens).`). + +| Modality | Cost | Max, single modality | +| :-- | :-- | :-- | +| Text | 1 token per subword | 8,192 tokens | +| Image | up to 280 tokens per image | about 29 images | +| Video | up to 140 tokens per sampled frame | about 58 frames | +| Audio | 25 tokens per second | about 327 seconds | + +Video is sampled at 1 frame per second and capped at 32 frames by default, spread uniformly over longer clips, so a video of any length fits by default. A 2-minute 720p clip, for example, uses 3,906 tokens. Only the sampled frames are decoded. + +To change the sampling rate or frame cap for every request, use `--mm-process-config`: + +```bash +sglang serve \ + --model-path google/embeddinggemma-2 \ + --mm-process-config '{"video": {"fps": 2, "max_frames": 48}}' \ + --host 0.0.0.0 +``` + +The number of frames that fit depends on resolution. At 720p, 64 frames fit; 70 frames exceed the context. diff --git a/docs/docs.json b/docs/docs.json index 3367c8fbfc..a83abc673e 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1339,6 +1339,7 @@ "group": "Google", "pages": [ "cookbook/autoregressive/Google/EmbeddingGemma", + "cookbook/autoregressive/Google/EmbeddingGemma2", "cookbook/autoregressive/Google/Gemma4", "cookbook/autoregressive/Google/DiffusionGemma" ] diff --git a/python/sglang/kernels/ops/attention/extend_attention.py b/python/sglang/kernels/ops/attention/extend_attention.py index 94e8f43989..fc810aa5f1 100644 --- a/python/sglang/kernels/ops/attention/extend_attention.py +++ b/python/sglang/kernels/ops/attention/extend_attention.py @@ -698,10 +698,18 @@ def _fwd_kernel( final_mask &= mask_non_causal if SLIDING_WINDOW_SIZE > 0: - # Add mask where q_id <= kv_id + sliding_window_size - window_mask = (cur_block_m * BLOCK_M + offs_m[:, None]) <= ( - start_n + offs_n[None, :] + SLIDING_WINDOW_SIZE - ) + if not IS_CAUSAL: + window_mask = ( + (cur_block_m * BLOCK_M + offs_m[:, None]) + <= (start_n + offs_n[None, :] + SLIDING_WINDOW_SIZE) + ) & ( + (start_n + offs_n[None, :]) + <= (cur_block_m * BLOCK_M + offs_m[:, None] + SLIDING_WINDOW_SIZE) + ) + else: + window_mask = (cur_block_m * BLOCK_M + offs_m[:, None]) <= ( + start_n + offs_n[None, :] + SLIDING_WINDOW_SIZE + ) final_mask &= window_mask SKIP_TILE = False @@ -1312,6 +1320,8 @@ def _fwd_kernel_unified( # Sliding window: query can attend to keys within window_size window_mask = q_abs_pos <= (k_abs_pos + SLIDING_WINDOW_SIZE) + if not IS_CAUSAL: + window_mask &= k_abs_pos <= (q_abs_pos + SLIDING_WINDOW_SIZE) final_mask &= window_mask # Check if we can skip this tile diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index 7d974e5381..dc1a9be316 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -836,6 +836,24 @@ def handle_model_capability_adjustments(server_args: Any): logger.info( "Embedding architecture detected: enabling embedding mode automatically." ) + if ( + embedding_model_spec is not None + and embedding_model_spec.safe_disable_radix_cache + ): + declare_resolution( + server_args, + "_handle_model_capability_adjustments", + disable_radix_cache=True, + ) + if ( + embedding_model_spec is not None + and embedding_model_spec.safe_disable_chunked_prefill + ): + declare_resolution( + server_args, + "_handle_model_capability_adjustments", + chunked_prefill_size=-1, + ) is_embedding_gemma = ( embedding_model_spec is not None diff --git a/python/sglang/srt/arg_groups/model_overrides/__init__.py b/python/sglang/srt/arg_groups/model_overrides/__init__.py index b0be0a9556..bc29d336db 100644 --- a/python/sglang/srt/arg_groups/model_overrides/__init__.py +++ b/python/sglang/srt/arg_groups/model_overrides/__init__.py @@ -12,6 +12,7 @@ from sglang.srt.arg_groups.model_overrides import bailing_moe_v3 # noqa: F401 from sglang.srt.arg_groups.model_overrides import cohere2_moe # noqa: F401 from sglang.srt.arg_groups.model_overrides import deepseek_v2 # noqa: F401 from sglang.srt.arg_groups.model_overrides import deepseek_v4 # noqa: F401 +from sglang.srt.arg_groups.model_overrides import embedding_gemma2 # noqa: F401 from sglang.srt.arg_groups.model_overrides import ernie45_vl # noqa: F401 from sglang.srt.arg_groups.model_overrides import exaone # noqa: F401 from sglang.srt.arg_groups.model_overrides import falcon_h1 # noqa: F401 diff --git a/python/sglang/srt/arg_groups/model_overrides/embedding_gemma2.py b/python/sglang/srt/arg_groups/model_overrides/embedding_gemma2.py new file mode 100644 index 0000000000..af4ac00bda --- /dev/null +++ b/python/sglang/srt/arg_groups/model_overrides/embedding_gemma2.py @@ -0,0 +1,43 @@ +# Copyright 2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Config-time override declarations for EmbeddingGemma2. + +Architectures: EmbeddingGemma2Model. +Sets disable_radix_cache=True and default attention_backend="triton" for EmbeddingGemma2. +""" + +import logging +from typing import Any + +from sglang.srt.arg_groups.model_override_base import ( + _register_for, + is_attention_backend_not_set, + resolving_view, +) + +logger = logging.getLogger(__name__) + + +@_register_for("EmbeddingGemma2Model") +def _embedding_gemma2_overrides(server_args: Any, hf_config: Any) -> dict[str, Any]: + cfg = resolving_view(server_args) + overrides: dict[str, Any] = { + "disable_radix_cache": True, + } + if is_attention_backend_not_set(cfg) or cfg.attention_backend is None: + logger.info("Use triton as default attention backend for EmbeddingGemma2Model") + overrides["attention_backend"] = "triton" + + return overrides diff --git a/python/sglang/srt/configs/embedding_model_spec.py b/python/sglang/srt/configs/embedding_model_spec.py index 6bcf5a326c..20867c8b6b 100644 --- a/python/sglang/srt/configs/embedding_model_spec.py +++ b/python/sglang/srt/configs/embedding_model_spec.py @@ -5,9 +5,10 @@ implementations, documentation, and benchmarks can consume the same contract without each reimplementing a partial list of embedding architectures. """ +from collections.abc import Sequence from dataclasses import dataclass from enum import Enum -from typing import Any, Sequence +from typing import Any class EmbeddingTask(str, Enum): @@ -168,9 +169,37 @@ def embedding_support_matrix() -> list[dict[str, Any]]: **_embedding_gemma_spec().as_dict(), } ) + rows.append( + { + "architecture": "EmbeddingGemma2Model", + **_embedding_gemma2_spec().as_dict(), + } + ) return rows +def _embedding_gemma2_spec() -> EmbeddingModelSpec: + return EmbeddingModelSpec( + family="embedding_gemma2", + task=EmbeddingTask.EMBED, + execution=EmbeddingExecution.MULTIMODAL, + attention=AttentionPattern.BIDIRECTIONAL, + pooling=PoolingStrategy.MEAN, + normalize=True, + postprocessor="model_defined", + tokenizer_special_tokens="model_default", + supports_dimensions=False, + supports_token_embeddings=False, + supports_multimodal=True, + requires_embedding_flag=False, + auto_enable_embedding=True, + bidirectional_attention=True, + bcg_prefill_policy=BCGPrefillPolicy.DEFAULT, + safe_disable_radix_cache=True, + safe_disable_chunked_prefill=True, + ) + + def _embedding_gemma_spec() -> EmbeddingModelSpec: return EmbeddingModelSpec( family="embeddinggemma", @@ -276,6 +305,9 @@ def resolve_embedding_model_spec( architecture_set = set(architectures or ()) + if "EmbeddingGemma2Model" in architecture_set: + return _embedding_gemma2_spec() + if is_embedding_gemma: return _embedding_gemma_spec() diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 2d0d3eea11..37cbbe45ff 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -1119,6 +1119,7 @@ class ModelConfig: "MiMoV2MTP", "Gemma4ForCausalLM", "Gemma4ForConditionalGeneration", + "EmbeddingGemma2Model", "InklingForConditionalGeneration", "InklingForConditionalGenerationMTP", "Gemma4UnifiedForConditionalGeneration", @@ -1187,7 +1188,13 @@ class ModelConfig: else: self.context_len = context_length else: - self.context_len = derived_context_len + if ( + self.hf_config.architectures + and self.hf_config.architectures[0] == "EmbeddingGemma2Model" + ): + self.context_len = min(derived_context_len, 8192) + else: + self.context_len = derived_context_len # Transfer context_len to HuggingFace config so models can access it self.hf_config.context_len = self.context_len @@ -2241,6 +2248,7 @@ def is_generation_model(model_architectures: List[str], is_embedding: bool = Fal or "XLMRobertaModel" in model_architectures or "XLMRobertaForSequenceClassification" in model_architectures or "Gemma2ForSequenceClassification" in model_architectures + or "EmbeddingGemma2Model" in model_architectures or "Lfm2BidirectionalModel" in model_architectures ): return False @@ -2258,6 +2266,7 @@ multimodal_model_archs = [ "Gemma3ForConditionalGeneration", "Gemma3nForConditionalGeneration", "Gemma4ForConditionalGeneration", + "EmbeddingGemma2Model", "Gemma4UnifiedForConditionalGeneration", "DiffusionGemmaForBlockDiffusion", "Glm4vForConditionalGeneration", @@ -2519,6 +2528,7 @@ def is_hybrid_swa_model( "Step3p7ForConditionalGeneration", "Gemma4ForCausalLM", "Gemma4ForConditionalGeneration", + "EmbeddingGemma2Model", "Gemma4UnifiedForConditionalGeneration", "DiffusionGemmaForBlockDiffusion", "LagunaForCausalLM", @@ -2600,6 +2610,7 @@ def get_hybrid_layer_ids( elif ( "Gemma4ForCausalLM" in model_architectures or "Gemma4ForConditionalGeneration" in model_architectures + or "EmbeddingGemma2Model" in model_architectures or "Gemma4UnifiedForConditionalGeneration" in model_architectures or "DiffusionGemmaForBlockDiffusion" in model_architectures or "LagunaForCausalLM" in model_architectures diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 9629b7aab6..3ba6045ff6 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -1271,10 +1271,16 @@ class EmbeddingReqInput: ) def normalize_batch_and_arguments(self): - # at least one of text, input_ids, or image should be provided - if self.text is None and self.input_ids is None and self.image_data is None: + # at least one of text, input_ids, image, video, or audio should be provided + if ( + self.text is None + and self.input_ids is None + and self.image_data is None + and self.video_data is None + and self.audio_data is None + ): raise ValueError( - "At least one of text, input_ids, or image should be provided" + "At least one of text, input_ids, image, video, or audio should be provided" ) # text and input_ids cannot be provided at the same time @@ -1288,18 +1294,37 @@ class EmbeddingReqInput: # check the batch size of text if self.text is not None: if isinstance(self.text, list): - self.batch_size += len(self.text) + self.batch_size = len(self.text) self.is_single = False else: - self.batch_size += 1 - - # check the batch size of input_ids - if self.input_ids is not None: + self.batch_size = 1 + elif self.input_ids is not None: if isinstance(self.input_ids[0], list): - self.batch_size += len(self.input_ids) + self.batch_size = len(self.input_ids) self.is_single = False else: - self.batch_size += 1 + self.batch_size = 1 + else: + # Without text, a flat media list is one request carrying all items + # (the HF processor reading); only a list of per-request lists is a batch. + media = next( + data + for data in (self.image_data, self.video_data, self.audio_data) + if data is not None + ) + if ( + isinstance(media, list) + and len(media) > 0 + and all(isinstance(item, list) for item in media) + ): + self.batch_size = len(media) + self.is_single = False + else: + self.batch_size = 1 + self.is_single = True + + if not self.is_single: + self._validate_mm_batch_lengths() # Fill in default arguments if self.is_single: @@ -1325,6 +1350,19 @@ class EmbeddingReqInput: self._validate_rid_uniqueness() + def _validate_mm_batch_lengths(self): + # A scalar media item is broadcast to every request; a list is per-request. + for field_name, data in ( + ("image_data", self.image_data), + ("video_data", self.video_data), + ("audio_data", self.audio_data), + ): + if isinstance(data, list) and len(data) != self.batch_size: + raise ValueError( + f"{field_name} has {len(data)} entries but the batch has " + f"{self.batch_size} requests; pass one entry (or None) per request." + ) + def _normalize_lora_paths(self, num): """Normalize LoRA paths for batch processing.""" if self.lora_path is not None: @@ -1361,6 +1399,17 @@ class EmbeddingReqInput: if i in cache: return cache[i] + # Lengths were checked in _validate_mm_batch_lengths; scalars broadcast. + image_item = ( + self.image_data[i] if isinstance(self.image_data, list) else self.image_data + ) + video_item = ( + self.video_data[i] if isinstance(self.video_data, list) else self.video_data + ) + audio_item = ( + self.audio_data[i] if isinstance(self.audio_data, list) else self.audio_data + ) + if self.is_cross_encoder_request: sub = EmbeddingReqInput( rid=self.rid[i], @@ -1390,9 +1439,9 @@ class EmbeddingReqInput: rid=self.rid[i], text=self.text[i] if self.text is not None else None, input_ids=self.input_ids[i] if self.input_ids is not None else None, - image_data=self.image_data[i] if self.image_data is not None else None, - video_data=self.video_data[i] if self.video_data is not None else None, - audio_data=self.audio_data[i] if self.audio_data is not None else None, + image_data=image_item, + video_data=video_item, + audio_data=audio_item, embed_override_token_id=self.embed_override_token_id, embed_overrides=( self.embed_overrides[i] diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 2ccae590ef..b3975bf3f8 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -987,7 +987,11 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): Tuple[List[int], Optional[List[int]]], Tuple[List[List[int]], Optional[List[List[int]]]], ]: - if not texts or self.tokenizer is None: + if ( + texts is None + or (isinstance(texts, list) and len(texts) == 0) + or self.tokenizer is None + ): raise ValueError("texts cannot be empty and tokenizer must be initialized") # Step 1: Detect input format and prepare for tokenization diff --git a/python/sglang/srt/models/embedding_gemma2.py b/python/sglang/srt/models/embedding_gemma2.py new file mode 100644 index 0000000000..579c4b87f2 --- /dev/null +++ b/python/sglang/srt/models/embedding_gemma2.py @@ -0,0 +1,963 @@ +# Copyright 2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""EmbeddingGemma2 model (EmbeddingGemma v2) for SGLang. + +Inference-only multimodal bidirectional embedding model compatible with HuggingFace. +Supports text, image, audio, and video inputs with mean pooling and unit L2 normalization. +""" + +import logging +from collections.abc import Iterable +from typing import Any, Callable, cast + +import torch +import torch.nn.functional as F +from torch import nn +from transformers import PretrainedConfig + +from sglang.srt.layers.activation import get_act_fn +from sglang.srt.layers.attention.vision import VisionSdpaAttention +from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm +from sglang.srt.layers.linear import ( + ColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from sglang.srt.layers.pooler import Pooler, PoolingType +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.radix_attention import AttentionType, RadixAttention +from sglang.srt.layers.rotary_embedding import get_rope +from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding +from sglang.srt.managers.mm_utils import ( + MultiModalityDataPaddingPatternMultimodalTokens, + general_mm_embed_routine, +) +from sglang.srt.managers.schedule_batch import ( + ForwardBatch, + Modality, + MultimodalDataItem, + MultimodalInputs, + flatten_nested_list, +) +from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.models.gemma4_audio import Gemma4AudioEncoder +from sglang.srt.models.gemma4_causal import Gemma4MLP +from sglang.srt.models.gemma4_mm import ( + Gemma4ForConditionalGeneration, + Gemma4MultimodalEmbedder, +) +from sglang.srt.models.gemma4_vision import Gemma4VisionEncoder +from sglang.srt.runtime_context import get_mm, get_parallel +from sglang.srt.utils import add_prefix + +logger = logging.getLogger(__name__) + + +class EmbeddingGemma2RMSNorm(nn.Module): + """Root Mean Square Layer Normalization with optional learnable scale. + + Used across EmbeddingGemma2 attention norms, projection norms, and decoder layer norms. + Supports forward_native binding for 3D/4D tensors and with_scale=False operations + (e.g., v_norm in self-attention layers where scaling is omitted). + """ + + def __init__(self, dim: int, eps: float = 1e-6, with_scale: bool = True): + super().__init__() + self.eps = eps + self.with_scale = with_scale + + if self.with_scale: + self.weight = nn.Parameter(torch.ones(dim), requires_grad=True) + + def _norm(self, hidden_states: torch.Tensor): + mean_squared = hidden_states.pow(2).mean(-1, keepdim=True) + self.eps + return hidden_states * torch.pow(mean_squared, -0.5) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + normed_output = self._norm(hidden_states.float()) + if self.with_scale: + normed_output = normed_output * self.weight.float() + return normed_output.type_as(hidden_states) + + def forward_native(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.forward(hidden_states) + + +class EmbeddingGemma2Attention(nn.Module): + """Attention layer for EmbeddingGemma2. + + Implements sliding window attention with an inclusive bidirectional window (<= 512) + and full global attention depending on config.layer_types. In sliding layers, + each token attends to all tokens within distance <= 512 both backward and forward, + unlike causal SWA where attention is restricted to past tokens. + """ + + def __init__( + self, + config: PretrainedConfig, + layer_idx: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.layer_idx = layer_idx + layer_type = config.layer_types[layer_idx] + self.is_sliding = layer_type == "sliding_attention" + + # Dims depend on layer type + if self.is_sliding: + self.head_dim = getattr(config, "swa_head_dim", 256) + self.num_kv_heads = getattr(config, "swa_num_key_value_heads", 2) + rope_theta = config.rope_parameters["sliding_attention"].get( + "rope_theta" + ) or config.rope_parameters["sliding_attention"].get("base", 10000.0) + self.sliding_window = 512 + else: + self.head_dim = getattr(config, "head_dim", 512) + self.num_kv_heads = getattr(config, "num_key_value_heads", 1) + rope_theta = config.rope_parameters["full_attention"].get( + "rope_theta" + ) or config.rope_parameters["full_attention"].get("base", 1000000.0) + self.sliding_window = -1 + + self.total_num_heads = config.num_attention_heads + self.total_num_kv_heads = self.num_kv_heads + tp_size = get_parallel().tp_size + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + + self.qkv_proj = QKVParallelLinear( + config.hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=getattr(config, "attention_bias", False), + quant_config=quant_config, + prefix=add_prefix("qkv_proj", prefix), + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + config.hidden_size, + bias=getattr(config, "attention_bias", False), + quant_config=quant_config, + prefix=add_prefix("o_proj", prefix), + ) + + self.q_norm = EmbeddingGemma2RMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = EmbeddingGemma2RMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.v_norm = EmbeddingGemma2RMSNorm( + self.head_dim, eps=config.rms_norm_eps, with_scale=False + ) + + max_pos = min( + config.max_position_embeddings, + getattr(config, "context_len", config.max_position_embeddings), + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_pos, + base=int(rope_theta), + is_neox_style=True, + ) + + self.attn = RadixAttention( + self.num_heads, + self.head_dim, + scaling=1.0, + num_kv_heads=self.num_kv_heads, + layer_id=layer_idx, + sliding_window_size=self.sliding_window, + quant_config=quant_config, + prefix=add_prefix("attn", prefix), + attn_type=AttentionType.ENCODER_ONLY, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + + q = q.unflatten(-1, (self.num_heads, self.head_dim)) + q = self.q_norm(q).flatten(-2, -1) + + k = k.unflatten(-1, (self.num_kv_heads, self.head_dim)) + k = self.k_norm(k).flatten(-2, -1) + + q, k = self.rotary_emb(positions, q, k) + + q = q.unflatten(-1, (self.num_heads, self.head_dim)) + k = k.unflatten(-1, (self.num_kv_heads, self.head_dim)) + + v = v.unflatten(-1, (self.num_kv_heads, self.head_dim)) + v = self.v_norm(v) + + attn_out = self.attn(q, k, v, forward_batch=forward_batch) + if attn_out.dim() == 3: + attn_out = attn_out.flatten(-2, -1) + out, _ = self.o_proj(attn_out) + return out + + +class EmbeddingGemma2PLEBlock(nn.Module): + def __init__( + self, + config: PretrainedConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.per_layer_input_gate = ReplicatedLinear( + config.hidden_size, + config.hidden_size_per_layer_input, + bias=False, + quant_config=quant_config, + prefix=add_prefix("per_layer_input_gate", prefix), + ) + self.per_layer_projection = ReplicatedLinear( + config.hidden_size_per_layer_input, + config.hidden_size, + bias=False, + quant_config=quant_config, + prefix=add_prefix("per_layer_projection", prefix), + ) + self.post_per_layer_input_norm = EmbeddingGemma2RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.act_fn = get_act_fn(config.hidden_activation) + + def forward( + self, hidden_states: torch.Tensor, per_layer_input: torch.Tensor + ) -> torch.Tensor: + gate = self.act_fn(self.per_layer_input_gate(hidden_states)[0]) + gated = gate * per_layer_input + proj = self.per_layer_projection(gated)[0] + return hidden_states + self.post_per_layer_input_norm(proj) + + +class EmbeddingGemma2DecoderLayer(nn.Module): + """Decoder layer for EmbeddingGemma2. + + Combines self-attention (with sliding or full attention depending on layer index), + feed-forward MLP, and Per-Layer Embedding (PLE) block, followed by scaling by a + persistent scalar parameter (`layer_scalar`). + """ + + def __init__( + self, + layer_idx: int, + config: PretrainedConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.self_attn = EmbeddingGemma2Attention( + config, layer_idx, quant_config, prefix=add_prefix("self_attn", prefix) + ) + self.mlp = Gemma4MLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_activation=config.hidden_activation, + quant_config=quant_config, + prefix=add_prefix("mlp", prefix), + ) + self.ple_block = EmbeddingGemma2PLEBlock( + config, quant_config=quant_config, prefix=add_prefix("ple_block", prefix) + ) + self.input_layernorm = EmbeddingGemma2RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.post_attention_layernorm = EmbeddingGemma2RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.pre_feedforward_layernorm = EmbeddingGemma2RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.post_feedforward_layernorm = EmbeddingGemma2RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.register_buffer("layer_scalar", torch.ones(1), persistent=True) + self.layer_scalar: torch.Tensor + + def forward( + self, + positions: torch.Tensor, + h: torch.Tensor, + forward_batch: ForwardBatch, + per_layer_input: torch.Tensor, + ) -> torch.Tensor: + r = h + h = ( + self.post_attention_layernorm( + self.self_attn( + positions, self.input_layernorm(h), forward_batch=forward_batch + ) + ) + + r + ) + r = h + h = ( + self.post_feedforward_layernorm(self.mlp(self.pre_feedforward_layernorm(h))) + + r + ) + h = self.ple_block(h, per_layer_input) + return h * self.layer_scalar + + +class EmbeddingGemma2TextPLE(nn.Module): + def __init__( + self, + config: PretrainedConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.num_layers = config.num_hidden_layers + self.hidden_dim = config.hidden_size_per_layer_input + self.per_layer_model_projection = ColumnParallelLinear( + config.hidden_size, + self.num_layers * self.hidden_dim, + bias=False, + gather_output=True, + quant_config=quant_config, + prefix=add_prefix("per_layer_model_projection", prefix), + ) + self.per_layer_model_projection_scale = config.hidden_size**-0.5 + self.per_layer_projection_norm = EmbeddingGemma2RMSNorm( + self.hidden_dim, eps=config.rms_norm_eps + ) + + def forward(self, inputs_embeds: torch.Tensor) -> torch.Tensor: + proj, _ = self.per_layer_model_projection(inputs_embeds) + proj = proj * self.per_layer_model_projection_scale + proj = proj.reshape(*inputs_embeds.shape[:-1], self.num_layers, self.hidden_dim) + return self.per_layer_projection_norm(proj) + + +class EmbeddingGemma2VocabParallelEmbedding(VocabParallelEmbedding): + """Embedding that replaces multimodal placeholder tokens with pad_token_id before lookup.""" + + def __init__( + self, + *args, + pad_token_id: int = 0, + mm_token_ids: tuple[int, ...] = (), + embed_scale: float = 1.0, + **kwargs, + ) -> None: + super().__init__(*args, **kwargs) + self.pad_token_id = pad_token_id + self.mm_token_ids = mm_token_ids + self.embed_scale_val = embed_scale + self.register_buffer("embed_scale", torch.tensor(embed_scale), persistent=False) + self.embed_scale: torch.Tensor + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + if self.mm_token_ids: + for mm_id in self.mm_token_ids: + if (input_ids == mm_id).any(): + input_ids = torch.where( + input_ids == mm_id, self.pad_token_id, input_ids + ) + return super().forward(input_ids) * self.embed_scale.to( + cast(torch.dtype, self.weight.dtype) + ) + + +class EmbeddingGemma2TextModel(nn.Module): + def __init__( + self, + config: PretrainedConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + mm_ids = tuple( + tid + for tid in ( + getattr(config, "image_token_id", 258880), + getattr(config, "audio_token_id", 258881), + getattr(config, "video_token_id", 258884), + ) + if tid is not None + ) + self.embed_tokens = EmbeddingGemma2VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + pad_token_id=config.pad_token_id, + mm_token_ids=mm_ids, + embed_scale=config.hidden_size**0.5, + quant_config=quant_config, + prefix=add_prefix("embed_tokens", prefix), + ) + self.ple = EmbeddingGemma2TextPLE( + config, quant_config=quant_config, prefix=add_prefix("ple", prefix) + ) + self.layers = nn.ModuleList( + [ + EmbeddingGemma2DecoderLayer( + i, + config, + quant_config=quant_config, + prefix=add_prefix(f"layers.{i}", prefix), + ) + for i in range(config.num_hidden_layers) + ] + ) + self.norm = EmbeddingGemma2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.embedding_projection = ReplicatedLinear( + config.hidden_size, + getattr(config, "embedding_dim", 768), + bias=False, + quant_config=quant_config, + prefix=add_prefix("embedding_projection", prefix), + ) + + def get_input_embeddings(self) -> nn.Module: + return self.embed_tokens + + def dtype(self) -> torch.dtype: + return cast(torch.dtype, self.embed_tokens.weight.dtype) + + @property + def device(self) -> torch.device: + return cast(torch.device, self.embed_tokens.weight.device) + + def forward( + self, + input_ids: torch.Tensor | None = None, + positions: torch.Tensor | None = None, + forward_batch: ForwardBatch | None = None, + input_embeds: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor: + if positions is None and forward_batch is not None: + positions = getattr(forward_batch, "positions", None) + if positions is None: + positions = kwargs.get("positions") + if input_embeds is None: + input_embeds = self.embed_tokens(input_ids) + per_layer = self.ple(input_embeds) + h = input_embeds + for i, layer in enumerate(self.layers): + h = layer(positions, h, forward_batch, per_layer[..., i, :]) + h = self.norm(h) + h, _ = self.embedding_projection(h) + return h + + +class EmbeddingGemma2VisionPooler(nn.Module): + """Vision pooler for EmbeddingGemma2. + + Isolated from Gemma4VisionPooler to prevent regressions in generative vision pipelines. + Performs 2D spatial average pooling over vision patch tokens into soft visual tokens + scaled by root_hidden_size (hidden_size ** 0.5) according to output_length. + """ + + def __init__(self, config): + super().__init__() + self.hidden_size = config.hidden_size + self.root_hidden_size = self.hidden_size**0.5 + + def _avg_pool_by_positions( + self, x: torch.Tensor, patch_positions: torch.Tensor, length: int + ) -> tuple[torch.Tensor, torch.Tensor]: + input_seq_len = x.shape[1] + k = int((input_seq_len // length) ** 0.5) + k_squared = k**2 + if k_squared * length != input_seq_len: + raise ValueError( + f"Cannot pool {x.shape} to {length}: {k=}^2 times {length=} must be {input_seq_len}." + ) + clamped_positions = patch_positions.clamp(min=0) + max_x = clamped_positions[..., 0].max(dim=-1, keepdim=True)[0] + 1 + kernel_idxs = torch.div(clamped_positions, k, rounding_mode="floor") + kernel_idxs = kernel_idxs[..., 0] + (max_x // k) * kernel_idxs[..., 1] + + weights = F.one_hot(kernel_idxs.long(), length).float() / k_squared + output = weights.transpose(1, 2) @ x.float() + mask = torch.logical_not((weights == 0).all(dim=1)) + return output, mask + + def forward( + self, + hidden_states: torch.Tensor, + patch_positions: torch.Tensor, + padding_positions: torch.Tensor, + output_length: int | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if output_length is None: + raise ValueError( + "output_length is required for EmbeddingGemma2VisionPooler" + ) + if output_length > hidden_states.shape[1]: + raise ValueError( + f"Cannot output more soft tokens (requested {output_length}) than there are patches" + f" ({hidden_states.shape[1]}). Change the value of `num_soft_tokens` when processing." + ) + length = output_length + if isinstance(length, (list, tuple)): + length = length[0] + orig_dtype = hidden_states.dtype + hidden_states = hidden_states.masked_fill(padding_positions.unsqueeze(-1), 0.0) + if hidden_states.shape[1] == length: + pooler_mask = ~padding_positions + else: + hidden_states, pooler_mask = self._avg_pool_by_positions( + hidden_states, patch_positions, length + ) + hidden_states = (hidden_states.float() * self.root_hidden_size).to(orig_dtype) + return hidden_states, pooler_mask + + +class EmbeddingGemma2Model(nn.Module): + """EmbeddingGemma2 multimodal embedding model. + + Supports text, image, audio, and video inputs with mean pooling and L2 normalization + over a 768-dim output embedding space. + + Architecture and design choices: + - Selective modality loading: vision and audio towers are conditionally loaded + based on server_args.limit_mm_data_per_request to save VRAM when only text, + image, audio, or video modalities are served. + - EmbeddingGemma2RMSNorm / forward_native rebinding: rebinds forward to forward_native + across Gemma4RMSNorm and RMSNorm modules to handle 3D/4D tensors and with_scale=False. + - EmbeddingGemma2VisionPooler: dedicated 2D spatial average pooler isolated from + generative Gemma4VisionPooler to prevent regressions in generation paths. + - Bidirectional sliding window: attention uses inclusive <= 512 bidirectional SWA + enabling full bidirectional cross-token representations for embedding tasks. + - Non-meta buffer materialization: materializes persistent buffers such as embed_scale + while preserving non-meta buffers (including scalar/tensor buffers with inf/-inf). + """ + + def __init__( + self, + config: PretrainedConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + text_config = config.text_config + if hasattr(text_config, "allow_global_per_layer_attribute_access"): + text_config.allow_global_per_layer_attribute_access = True + self.text_config = text_config + + try: + limit_mm = get_mm().limit_mm_data_per_request or {} + except Exception: # noqa: BLE001 + limit_mm = {} + + # Gated Vision Tower + has_vision = getattr(config, "vision_config", None) is not None and ( + limit_mm.get("image", 1) > 0 or limit_mm.get("video", 1) > 0 + ) + self.vision_tower: Gemma4VisionEncoder | None + self.embed_vision: Gemma4MultimodalEmbedder | None + self.audio_tower: Gemma4AudioEncoder | None + self.embed_audio: Gemma4MultimodalEmbedder | None + + if has_vision: + self.vision_tower = Gemma4VisionEncoder( + config.vision_config, + quant_config=quant_config, + prefix=add_prefix("vision_tower", prefix), + ) + self.vision_tower.pooler = EmbeddingGemma2VisionPooler(config.vision_config) # type: ignore[assignment] + for layer in self.vision_tower.encoder.layers: # type: ignore[union-attr] + layer.self_attn.qkv_backend = VisionSdpaAttention( # type: ignore[attr-defined,union-attr] + head_dim=config.vision_config.head_dim, + num_heads=layer.self_attn.num_heads_per_partition, # type: ignore[attr-defined,arg-type,union-attr] + num_kv_heads=layer.self_attn.num_kv_heads_per_partition, # type: ignore[attr-defined,arg-type,union-attr] + dropout=0.0, + flatten_batch=False, + softmax_in_single_precision=False, + softmax_scale=1.0, + ) + self.embed_vision = Gemma4MultimodalEmbedder( + config.vision_config, + text_config, + quant_config=quant_config, + prefix=add_prefix("embed_vision", prefix), + ) + else: + self.vision_tower = None + self.embed_vision = None + + # Gated Audio Tower + has_audio = ( + getattr(config, "audio_config", None) is not None + and limit_mm.get("audio", 1) > 0 + ) + if has_audio: + self.audio_tower = Gemma4AudioEncoder( + config.audio_config, + quant_config=quant_config, + prefix=add_prefix("audio_tower", prefix), + ) + self.embed_audio = Gemma4MultimodalEmbedder( + config.audio_config, + text_config, + quant_config=quant_config, + prefix=add_prefix("embed_audio", prefix), + ) + else: + self.audio_tower = None + self.embed_audio = None + + self.language_model = EmbeddingGemma2TextModel( + text_config, + quant_config=quant_config, + prefix=add_prefix("language_model", prefix), + ) + self.pooler = Pooler(PoolingType.MEAN, normalize=True) + + for m in self.modules(): + if isinstance(m, (Gemma4RMSNorm, RMSNorm)): + m.forward = m.forward_native # type: ignore[method-assign,assignment] + + def get_attention_sliding_window_size(self) -> int: + return 512 + + def get_input_embeddings(self) -> nn.Module: + return self.language_model.get_input_embeddings() + + def pad_input_ids( + self, input_ids: list[int], mm_inputs: MultimodalInputs + ) -> list[int]: + pattern = MultiModalityDataPaddingPatternMultimodalTokens() + return pattern.pad_input_tokens(input_ids, mm_inputs) + + def get_image_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor: + if self.vision_tower is None or self.embed_vision is None: + raise ValueError( + "Image inputs provided but vision tower is not loaded. " + "Ensure limit_mm_data_per_request allows image > 0." + ) + vt = self.vision_tower + all_embeds = [] + for item in items: + all_pixel_values = flatten_nested_list([item.feature]) + all_position_ids = flatten_nested_list( + [getattr(item, "image_position_ids", None)] + ) + for pv_idx, pv in enumerate(all_pixel_values): + if pv.dim() in (2, 3) and pv.shape[-1] == self.text_config.hidden_size: + all_embeds.append(pv.to(self.language_model.device)) + continue + if pv_idx >= len(all_position_ids) or all_position_ids[pv_idx] is None: + raise ValueError( + f"pixel_values[{pv_idx}] has no matching image_position_ids." + ) + pp = all_position_ids[pv_idx] + if pv.dim() == 2: + pv = pv.unsqueeze(0) + if pp.dim() == 2: + pp = pp.unsqueeze(0) + pv = pv.to(device=vt.device, dtype=self.language_model.dtype()) + pp = pp.to(device=vt.device) + + pooled, pooler_mask = vt(pv, pp) + for hs, mask in zip(pooled, pooler_mask): + real_tokens = hs[mask] + all_embeds.append( + self.embed_vision( + inputs_embeds=real_tokens.unsqueeze(0) + ).squeeze(0) + ) + if all_embeds: + return torch.cat(all_embeds, dim=0) + else: + return torch.empty( + 0, + self.text_config.hidden_size, + device=next(self.parameters()).device, + dtype=self.language_model.dtype(), + ) + + def get_video_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor: + if self.vision_tower is None or self.embed_vision is None: + raise ValueError( + "Video inputs provided but vision tower is not loaded. " + "Ensure limit_mm_data_per_request allows video > 0." + ) + vt = self.vision_tower + all_embeds = [] + for item in items: + all_pixel_values = flatten_nested_list([item.feature]) + all_position_ids = flatten_nested_list( + [getattr(item, "video_position_ids", None)] + ) + for pv_idx, pv in enumerate(all_pixel_values): + if pv.dim() in (2, 3) and pv.shape[-1] == self.text_config.hidden_size: + all_embeds.append(pv.to(self.language_model.device)) + continue + if pv_idx >= len(all_position_ids) or all_position_ids[pv_idx] is None: + raise ValueError( + f"pixel_values_videos[{pv_idx}] has no matching video_position_ids." + ) + pp = all_position_ids[pv_idx] + if pv.dim() == 4: + pv = pv.reshape(-1, pv.shape[-2], pv.shape[-1]) + if pp.dim() == 4: + pp = pp.reshape(-1, pp.shape[-2], pp.shape[-1]) + if pv.dim() == 2: + pv = pv.unsqueeze(0) + if pp.dim() == 2: + pp = pp.unsqueeze(0) + pv = pv.to(device=vt.device, dtype=self.language_model.dtype()) + pp = pp.to(device=vt.device) + + pooled, pooler_mask = vt(pv, pp) + for hs, mask in zip(pooled, pooler_mask): + real_tokens = hs[mask] + all_embeds.append( + self.embed_vision( + inputs_embeds=real_tokens.unsqueeze(0) + ).squeeze(0) + ) + if all_embeds: + return torch.cat(all_embeds, dim=0) + else: + return torch.empty( + 0, + self.text_config.hidden_size, + device=next(self.parameters()).device, + dtype=self.language_model.dtype(), + ) + + def get_audio_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor: + if self.audio_tower is None or self.embed_audio is None: + raise ValueError( + "Audio inputs provided but audio tower is not loaded. " + "Ensure limit_mm_data_per_request allows audio > 0." + ) + all_input_features = flatten_nested_list([item.feature for item in items]) + all_input_features_mask = flatten_nested_list( + [~item.input_features_mask for item in items] # type: ignore[operator] + ) + all_embeds = [] + for input_features, input_features_mask in zip( + all_input_features, all_input_features_mask + ): + if input_features.dim() == 2: + input_features = input_features.unsqueeze(0) + if input_features_mask.dim() == 1: + input_features_mask = input_features_mask.unsqueeze(0) + + input_features = input_features.to( + device=self.audio_tower.device, + dtype=self.language_model.dtype(), + ) + input_features_mask = input_features_mask.to(device=input_features.device) + + audio_encodings, audio_mask = self.audio_tower( + input_features, input_features_mask + ) + audio_features = self.embed_audio(inputs_embeds=audio_encodings) + for enc, mask in zip(audio_features, audio_mask): + all_embeds.append(enc[~mask]) + if all_embeds: + return torch.cat(all_embeds, dim=0) + else: + return torch.empty( + 0, + self.text_config.hidden_size, + device=next(self.parameters()).device, + dtype=self.language_model.dtype(), + ) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: torch.Tensor | None = None, + get_embedding: bool = True, + **kwargs, + ): + data_embedding_funcs: dict[ + Modality, Callable[[list[MultimodalDataItem]], Any] + ] = {} + if self.vision_tower is not None: + data_embedding_funcs[Modality.IMAGE] = self.get_image_feature + data_embedding_funcs[Modality.VIDEO] = self.get_video_feature + if self.audio_tower is not None: + data_embedding_funcs[Modality.AUDIO] = self.get_audio_feature + + if forward_batch.contains_mm_inputs(): + h = general_mm_embed_routine( + input_ids=input_ids, + forward_batch=forward_batch, + language_model=self.language_model, + data_embedding_funcs=data_embedding_funcs, + placeholder_tokens={ + Modality.IMAGE: [getattr(self.config, "image_token_id", 258880)], + Modality.AUDIO: [getattr(self.config, "audio_token_id", 258881)], + Modality.VIDEO: [getattr(self.config, "video_token_id", 258884)], + }, + positions=positions, + ) + else: + h = self.language_model( + input_ids=input_ids, + positions=positions, + forward_batch=forward_batch, + input_embeds=input_embeds, + ) + + # Pooler expects fp32 tensor, mean pool, L2 normalize + return self.pooler(h.float(), forward_batch) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> list[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + buffers_dict = dict(self.named_buffers()) + loaded_params = set() + + for name, loaded_weight in weights: + # Tower skipping + if self.audio_tower is None and ( + name.startswith(("audio_tower.", "embed_audio.")) + ): + continue + if self.vision_tower is None and ( + name.startswith(("vision_tower.", "embed_vision.")) + ): + continue + + # Tower remapping using Gemma4 static methods + if name.startswith("audio_tower."): + name = Gemma4ForConditionalGeneration._remap_audio_tower_name(name) + if name.startswith( + ("vision_tower.", "audio_tower.", "embed_vision.", "embed_audio.") + ): + name = Gemma4ForConditionalGeneration._remap_tower_name( + name, params_dict + ) + + # Persistent buffer loading (e.g. layer_scalar) + if name in buffers_dict: + buffers_dict[name].copy_(loaded_weight) + loaded_params.add(name) + continue + + # Stacked projection routing + for param_name, shard_name, shard_id in stacked_params_mapping: + if shard_name in name: + mapped_name = name.replace(shard_name, param_name) + if mapped_name in params_dict: + param = params_dict[mapped_name] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + try: + weight_loader(param, loaded_weight, shard_id) # type: ignore[call-arg] + except TypeError: + weight_loader(param, loaded_weight) + loaded_params.add(mapped_name) + break + else: + if name in params_dict: + param = params_dict[name] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, loaded_weight) + loaded_params.add(name) + # Audit unloaded parameters + expected_params = set(params_dict.keys()) + if self.vision_tower is None: + expected_params = { + k + for k in expected_params + if not k.startswith(("vision_tower.", "embed_vision.")) + } + if self.audio_tower is None: + expected_params = { + k + for k in expected_params + if not k.startswith(("audio_tower.", "embed_audio.")) + } + unloaded = expected_params - loaded_params + if unloaded: + logger.warning( + f"EmbeddingGemma2Model: {len(unloaded)} expected parameters were not loaded: {sorted(unloaded)[:10]}" + ) + + # Materialize non-persistent / meta buffers + target_device = self.language_model.device + target_dtype = self.language_model.dtype() + for module in self.modules(): + for name, buf in list(module.named_buffers(recurse=False)): + if buf is not None and (buf.is_meta or buf.device != target_device): + if name == "embed_scale": + scale_val = getattr( + module, + "embed_scale_val", + getattr(self.config, "hidden_size", 512) ** 0.5, + ) + module._buffers[name] = torch.tensor( + scale_val, dtype=target_dtype, device=target_device + ) + elif not buf.is_meta: + module._buffers[name] = buf.to( + device=target_device, dtype=target_dtype + ) + else: + module._buffers[name] = torch.ones( + buf.shape, dtype=target_dtype, device=target_device + ) + + for m in self.modules(): + if isinstance(m, (Gemma4RMSNorm, RMSNorm)): + m.forward = m.forward_native # type: ignore[method-assign,assignment] + + return list(loaded_params) + + def get_weights_by_name( + self, name: str, truncate_size: int = 100, tp_size: int = 1 + ) -> torch.Tensor | None: + params_dict = dict(self.named_parameters()) + if name in params_dict: + return ( + params_dict[name] + .data.cpu() + .to(torch.float32) + .numpy() + .tolist()[:truncate_size] + ) + return None + + +EntryClass = [EmbeddingGemma2Model] diff --git a/python/sglang/srt/multimodal/processors/embedding_gemma2.py b/python/sglang/srt/multimodal/processors/embedding_gemma2.py new file mode 100644 index 0000000000..731673c060 --- /dev/null +++ b/python/sglang/srt/multimodal/processors/embedding_gemma2.py @@ -0,0 +1,276 @@ +# Copyright 2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Multimodal processor for EmbeddingGemma2Model (EmbeddingGemma v2).""" + +import logging +import re +from typing import Any + +import numpy as np +import torch +from transformers.video_utils import VideoMetadata + +from sglang.srt.models.embedding_gemma2 import EmbeddingGemma2Model +from sglang.srt.multimodal.processors.base_processor import ( + Modality, + MultimodalSpecialTokens, +) +from sglang.srt.multimodal.processors.gemma4 import Gemma4SGLangProcessor +from sglang.srt.utils.common import load_audio +from sglang.srt.utils.video_decoder import VideoDecoderWrapper + +logger = logging.getLogger(__name__) + + +def _load_audio_from_memory( + audio_file: Any, sr: int | None = 16000, mono: bool = True +) -> np.ndarray: + """Load or resample audio waveforms from in-memory numpy array, torch tensor, tuple, or path.""" + if sr is None: + sr = 16000 + + input_sr = None + if isinstance(audio_file, tuple) and len(audio_file) == 2: + audio_file, input_sr = audio_file + + if isinstance(audio_file, torch.Tensor): + audio_file = audio_file.detach().cpu().numpy() + + if isinstance(audio_file, np.ndarray): + waveform = audio_file.astype(np.float32, copy=False) + if waveform.ndim > 1 and mono: + if waveform.shape[0] <= 8 and waveform.shape[0] < waveform.shape[-1]: + waveform = waveform.mean(axis=0) + else: + waveform = waveform.mean(axis=-1) + if ( + input_sr is not None + and sr is not None + and input_sr != sr + and len(waveform) > 0 + ): + try: + import torchaudio # type: ignore[import-untyped] + + audio_tensor = torch.from_numpy(waveform).float() + resampler = torchaudio.transforms.Resample( + orig_freq=input_sr, new_freq=sr + ) + waveform = resampler(audio_tensor).numpy().astype(np.float32) + except Exception: # noqa: BLE001 + from scipy import signal # type: ignore[import-untyped] + + num_samples = int(len(waveform) * float(sr) / input_sr) + waveform = signal.resample(waveform, num_samples).astype(np.float32) + return waveform + + return load_audio(audio_file, sr=sr, mono=mono) + + +class EmbeddingGemma2SGLangProcessor(Gemma4SGLangProcessor): + """Multimodal input processor for EmbeddingGemma2Model. + + Preprocesses image, video, and audio inputs for EmbeddingGemma v2: + - Prompt retention: retains prompt task/title prefixes verbatim, disabling the + anchored prompt suppression used in generative Gemma4. + - Media-only requests: synthesizes the placeholder prompt the HF processor builds + when no text is given, and rejects placeholder/media count mismatches. + - Video processing: samples frame indices with the HF video processor's + fps/max_frames rule before decoding, so only sampled frames are materialized. + - Audio processing: provides unpadded raw waveforms (16kHz mono float32). + """ + + models: list[Any] = [EmbeddingGemma2Model] # type: ignore[assignment] # noqa: RUF012 + + # Plain nvJPEG skips fancy chroma upsampling and drifts from the PIL decode the + # checkpoint was validated with (embedding cos ~0.99 vs HF); nvjpeg_fancy matches it. + gpu_image_decode = "nvjpeg_fancy" + + def __init__(self, hf_config, server_args, _processor, *args, **kwargs): + super().__init__(hf_config, server_args, _processor, *args, **kwargs) + self.disable_fast_image_processor = True + + # Resolve special tokens + self.IM_START_TOKEN_ID = getattr(hf_config, "boi_token_id", 255999) + self.IM_END_TOKEN_ID = getattr(hf_config, "eoi_token_id", 258882) + self.IMAGE_TOKEN_ID = getattr(hf_config, "image_token_id", 258880) + self.AUDIO_TOKEN_ID = getattr(hf_config, "audio_token_id", 258881) + self.VIDEO_TOKEN_ID = getattr(hf_config, "video_token_id", 258884) + self.AUDIO_START_TOKEN_ID = getattr(hf_config, "boa_token_id", 256000) + self.AUDIO_END_TOKEN_ID = getattr( + hf_config, "eoa_token_id", getattr(hf_config, "eoa_token_index", 258883) + ) + + @classmethod + def _load_single_item( + cls, + data: Any, + modality: Modality, + frame_count_limit: int | None = None, + audio_sample_rate: int | None = None, + discard_alpha_channel: bool = True, + ) -> Any: + if modality == Modality.AUDIO: + return _load_audio_from_memory(data, sr=audio_sample_rate, mono=True) + return super()._load_single_item( + data, + modality=modality, + frame_count_limit=frame_count_limit, + audio_sample_rate=audio_sample_rate, + discard_alpha_channel=discard_alpha_channel, + ) + + async def load_mm_data( # type: ignore[override] + self, + prompt: Any, + multimodal_tokens: MultimodalSpecialTokens, + image_data: list[Any] | None = None, + video_data: list[Any] | None = None, + audio_data: list[Any] | None = None, + **kwargs: Any, + ): + # Resolve the prompt here rather than overriding process_mm_data_async, so + # requests keep Gemma4's route through the preprocessing worker pool. + prompt = self._resolve_prompt( + prompt, + n_image=len(image_data) if image_data else 0, + n_video=len(video_data) if video_data else 0, + n_audio=len(audio_data) if audio_data else 0, + ) + return await super().load_mm_data( + prompt, multimodal_tokens, image_data, video_data, audio_data, **kwargs + ) + + def _resolve_prompt( + self, input_text: Any, *, n_image: int, n_video: int, n_audio: int + ) -> Any: + if not input_text: + # Same layout HF builds for text=None: images, then videos, then audio. + tokens = self.mm_tokens + return " ".join( + [tokens.image_token] * n_image + + [tokens.video_token] * n_video + + [tokens.audio_token] * n_audio + ) + prompt = ( + self._tokenizer.decode(input_text) + if isinstance(input_text, list) + else input_text + ) + found = {Modality.IMAGE: 0, Modality.VIDEO: 0, Modality.AUDIO: 0} + for part in re.split(self.mm_tokens.get_combined_regex(), prompt): + modality = self.mm_tokens.get_modality_of_token(part) + if modality is not None: + found[modality] += 1 + expected = { + Modality.IMAGE: n_image, + Modality.VIDEO: n_video, + Modality.AUDIO: n_audio, + } + # The generic loader silently drops unmatched media, which yields a wrong + # embedding instead of an error; HF raises here too. + mismatched = [ + f"{m.name.lower()}: {found[m]} placeholder(s) vs {expected[m]} item(s)" + for m in expected + if found[m] != expected[m] + ] + if mismatched: + raise ValueError( + "EmbeddingGemma2 needs exactly one <|image|>/<|video|>/<|audio|> " + "placeholder per media item, or no text at all; got " + + ", ".join(mismatched) + + "." + ) + return input_text + + def _sample_video(self, video: Any) -> tuple[torch.Tensor, VideoMetadata]: + """Decode only the frames the HF video processor would keep.""" + if isinstance(video, VideoDecoderWrapper): + total, src_fps = len(video), video.avg_fps + else: + frames, meta = video if isinstance(video, tuple) else (video, None) + if isinstance(meta, VideoMetadata): + src_fps = meta.fps + elif isinstance(meta, dict): + src_fps = meta.get("fps") + else: + src_fps = None + total = len(frames) + metadata = VideoMetadata( + total_num_frames=total, + fps=src_fps, + duration=total / src_fps if src_fps else None, + ) + video_processor = self._processor.video_processor + indices = video_processor.sample_frames( + metadata, + fps=self.video_config.get("fps", video_processor.fps), + max_frames=self.video_config.get("max_frames", video_processor.max_frames), + overflow_strategy=self.video_config.get( + "overflow_strategy", video_processor.overflow_strategy + ), + ).tolist() + if isinstance(video, VideoDecoderWrapper): + sampled = torch.as_tensor(video.get_frames_at(indices)).permute(0, 3, 1, 2) + elif isinstance(frames, torch.Tensor): + sampled = frames[indices] + else: + sampled = torch.as_tensor(np.asarray(frames)[indices]) + metadata.frames_indices = indices + return sampled.contiguous(), metadata + + def process_mm_data( # type: ignore[override] + self, + input_text: Any, + images: list[Any] | None = None, + videos: list[Any] | None = None, + audios: list[Any] | None = None, + **kwargs: Any, + ): + if "num_frames" in kwargs or "num_frames" in kwargs.get("videos_kwargs", {}): + raise ValueError( + "Sampling with `num_frames` is not supported for EmbeddingGemma2. " + "Use `fps` and `max_frames` instead." + ) + + # 1. Prompt retention: do NOT suppress task/title prefixes. + # input_text passes through verbatim. + + # 2. Audio: unpadded raw waveforms (16kHz mono float32) + if audios: + loaded_audios = [] + for a in audios: + if isinstance(a, (np.ndarray, torch.Tensor, tuple)): + loaded_audios.append( + _load_audio_from_memory(a, sr=16000, mono=True) + ) + else: + loaded_audios.append(np.asarray(a, dtype=np.float32)) + kwargs["audio"] = loaded_audios + kwargs.setdefault("audio_kwargs", {})["truncation"] = False + audios = None + + # 3. Video: frames are sampled here, so the HF processor must not resample. + if videos: + sampled = [self._sample_video(v) for v in videos] + videos = [frames for frames, _ in sampled] + kwargs.setdefault("videos_kwargs", {})["video_metadata"] = [ + meta for _, meta in sampled + ] + kwargs["do_sample_frames"] = False + + return super(Gemma4SGLangProcessor, self).process_mm_data( + input_text, images=images, videos=videos, audios=audios, **kwargs + ) diff --git a/python/sglang/srt/utils/hf_transformers/config.py b/python/sglang/srt/utils/hf_transformers/config.py index c082b7c454..b5a2386518 100644 --- a/python/sglang/srt/utils/hf_transformers/config.py +++ b/python/sglang/srt/utils/hf_transformers/config.py @@ -64,6 +64,7 @@ _GEMMA4_MODEL_TYPES = ( "gemma4_unified", "gemma4_unified_assistant", "diffusion_gemma", + "embedding_gemma2", ) diff --git a/test/registered/unit/configs/test_embedding_model_spec.py b/test/registered/unit/configs/test_embedding_model_spec.py index 23dff18784..5326630783 100644 --- a/test/registered/unit/configs/test_embedding_model_spec.py +++ b/test/registered/unit/configs/test_embedding_model_spec.py @@ -52,7 +52,9 @@ class TestEmbeddingModelSpec(unittest.TestCase): matrix = embedding_support_matrix() by_architecture = {row["architecture"]: row for row in matrix} - self.assertEqual(len(matrix), 7) + self.assertEqual(len(matrix), 8) + self.assertIn("EmbeddingGemma2Model", by_architecture) + self.assertTrue(by_architecture["EmbeddingGemma2Model"]["supports_multimodal"]) self.assertEqual(by_architecture["BertModel"]["family"], "bert") self.assertEqual(by_architecture["BertModel"]["attention"], "bidirectional") self.assertTrue(by_architecture["CLIPModel"]["supports_multimodal"]) @@ -117,6 +119,22 @@ class TestEmbeddingModelSpec(unittest.TestCase): self.assertEqual(spec.task, EmbeddingTask.NONE) self.assertEqual(spec.family, "none") + def test_embedding_gemma2_declares_multimodal_spec(self): + spec = resolve_embedding_model_spec( + ["EmbeddingGemma2Model"], + is_embedding_requested=False, + is_embedding_gemma=False, + ) + + self.assertEqual(spec.family, "embedding_gemma2") + self.assertEqual(spec.task, EmbeddingTask.EMBED) + self.assertEqual(spec.execution, EmbeddingExecution.MULTIMODAL) + self.assertEqual(spec.pooling, PoolingStrategy.MEAN) + self.assertTrue(spec.supports_multimodal) + self.assertFalse(spec.requires_embedding_flag) + self.assertTrue(spec.auto_enable_embedding) + self.assertEqual(spec.attention, AttentionPattern.BIDIRECTIONAL) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/managers/test_io_struct.py b/test/registered/unit/managers/test_io_struct.py index 59bb3c9283..b1e636c1da 100644 --- a/test/registered/unit/managers/test_io_struct.py +++ b/test/registered/unit/managers/test_io_struct.py @@ -1352,6 +1352,39 @@ class TestEmbeddingReqInputGetItem(CustomTestCase): with self.assertRaisesRegex(ValueError, "must match batch size"): req.normalize_batch_and_arguments() + def test_media_only_flat_list_is_one_request(self): + """Without text, a flat media list is one request with all items, as in HF.""" + req = EmbeddingReqInput(image_data=["a.jpg", "b.jpg"]) + req.normalize_batch_and_arguments() + self.assertTrue(req.is_single) + self.assertEqual(req.image_data, ["a.jpg", "b.jpg"]) + + def test_media_only_nested_list_is_a_batch(self): + """A list of per-request media lists is the explicit batch form.""" + req = EmbeddingReqInput(image_data=[["a.jpg"], ["b.jpg", "c.jpg"]]) + req.normalize_batch_and_arguments() + self.assertFalse(req.is_single) + self.assertEqual( + [req[0].image_data, req[1].image_data], [["a.jpg"], ["b.jpg", "c.jpg"]] + ) + + def test_media_list_pairs_with_text_batch(self): + """A media list pairs by position; a scalar item broadcasts to every request.""" + req = EmbeddingReqInput( + text=["t0", "t1"], image_data=["a.jpg", None], audio_data="s.wav" + ) + req.normalize_batch_and_arguments() + self.assertEqual( + [(req[i].image_data, req[i].audio_data) for i in range(2)], + [("a.jpg", "s.wav"), (None, "s.wav")], + ) + + def test_media_list_length_must_match_batch(self): + """A short media list must not be re-attached to the remaining requests.""" + req = EmbeddingReqInput(text=["t0", "t1", "t2"], image_data=["a.jpg"]) + with self.assertRaisesRegex(ValueError, "image_data has 1 entries"): + req.normalize_batch_and_arguments() + if __name__ == "__main__": unittest.main()