[Fix] nemotron_vl: Pad visual spans using processor offsets (#44581)
Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
This commit is contained in:
@@ -30,6 +30,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
general_mm_embed_routine,
|
||||
pad_mm_token_pairs,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -139,20 +140,18 @@ class NemotronH_Nano_VL_V2(EVS):
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
im_start_id: int = mm_inputs.im_start_id
|
||||
im_end_id: int = mm_inputs.im_end_id
|
||||
|
||||
visual_items = [item for item in mm_inputs.mm_items if not item.is_audio()]
|
||||
audio_items = [item for item in mm_inputs.mm_items if item.is_audio()]
|
||||
|
||||
all_data_offsets = []
|
||||
|
||||
if visual_items:
|
||||
# Literal boundary tokens in text must not change which spans are images.
|
||||
mm_inputs.mm_items = visual_items
|
||||
input_ids = pad_mm_token_pairs(
|
||||
input_ids, mm_inputs, data_token_pairs=[(im_start_id, im_end_id)]
|
||||
input_ids = pad_mm_tokens(input_ids, mm_inputs)
|
||||
all_data_offsets.extend(
|
||||
start - 1 for item in visual_items for start, _ in item.offsets
|
||||
)
|
||||
all_data_offsets.extend(mm_inputs.data_offsets)
|
||||
|
||||
audio_start_id = getattr(mm_inputs, "audio_start_id", None)
|
||||
audio_end_id = getattr(mm_inputs, "audio_end_id", None)
|
||||
|
||||
@@ -99,15 +99,20 @@ class TestNemotronHOmniVideo(CustomTestCase):
|
||||
def test_evs_keeps_image_pad_values_for_redistribution(self):
|
||||
"""EVS placeholder redistribution must not restore raw image tokens next to a video."""
|
||||
start, end, image_token = 1, 2, 18
|
||||
image = MultimodalDataItem(modality=Modality.IMAGE, offsets=[], hash=11)
|
||||
image = MultimodalDataItem(modality=Modality.IMAGE, offsets=[(2, 2)], hash=11)
|
||||
video = _video_item([2], pre_chunked_input_ids=[])
|
||||
video.offsets = [(5, 5)]
|
||||
video.hash = 22
|
||||
for item in (image, video):
|
||||
item.set_pad_value()
|
||||
input_ids = array("q", [7, start, image_token, end, start, image_token, end])
|
||||
video.pre_chunked_input_ids = input_ids.tolist()
|
||||
mm_inputs = MultimodalInputs(
|
||||
mm_items=[image, video], im_start_id=start, im_end_id=end
|
||||
mm_items=[image, video],
|
||||
im_start_id=start,
|
||||
im_end_id=end,
|
||||
im_token_id=image_token,
|
||||
video_token_id=image_token,
|
||||
)
|
||||
|
||||
model = object.__new__(NemotronH_Omni_Reasoning_V3)
|
||||
|
||||
@@ -2,11 +2,14 @@
|
||||
|
||||
import asyncio
|
||||
import unittest
|
||||
from array import array
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
|
||||
from sglang.srt.models.nano_nemotron_vl import NemotronH_Nano_VL_V2
|
||||
from sglang.srt.multimodal.evs import EVSConfig, EVSProcessor
|
||||
from sglang.srt.multimodal.processors.nano_nemotron_vl import (
|
||||
NanoNemotronVLImageProcessor,
|
||||
@@ -135,6 +138,44 @@ class TestNanoNemotronVLProcessor(CustomTestCase):
|
||||
)
|
||||
processor.load_mm_data.assert_not_awaited()
|
||||
|
||||
def test_visual_offsets_ignore_literal_boundaries_and_preserve_audio(self):
|
||||
model = object.__new__(NemotronH_Nano_VL_V2)
|
||||
for text, start in (
|
||||
([5, 19, 18, 18, 20, 7, 20], 2),
|
||||
([5, 19, 9, 19, 18, 18, 20, 7], 4),
|
||||
([19, 4, 4, 20, 5, 19, 18, 18, 20], 6),
|
||||
):
|
||||
with self.subTest(text=text):
|
||||
mm = SimpleNamespace(
|
||||
mm_items=[
|
||||
MultimodalDataItem(
|
||||
modality=Modality.IMAGE,
|
||||
offsets=[(start, start + 1)],
|
||||
pad_value=1_000_001,
|
||||
),
|
||||
MultimodalDataItem(
|
||||
modality=Modality.AUDIO,
|
||||
offsets=[(len(text) + 1, len(text) + 2)],
|
||||
pad_value=1_000_002,
|
||||
),
|
||||
],
|
||||
data_offsets=[],
|
||||
im_start_id=19,
|
||||
im_end_id=20,
|
||||
im_token_id=18,
|
||||
video_token_id=18,
|
||||
audio_token_id=32,
|
||||
audio_start_id=30,
|
||||
audio_end_id=31,
|
||||
)
|
||||
padded = model.pad_input_ids(array("q", text + [30, 32, 32, 31]), mm)
|
||||
expected = text[:]
|
||||
expected[start : start + 2] = [1_000_001] * 2
|
||||
self.assertEqual(
|
||||
list(padded), expected + [30, 1_000_002, 1_000_002, 31]
|
||||
)
|
||||
self.assertEqual(mm.data_offsets, [start - 1, len(text)])
|
||||
|
||||
def test_tubelet_label_matches_hf_reference(self):
|
||||
"""Multi-frame tubelet labels must match the HF processor's prompt format."""
|
||||
processor = object.__new__(NanoNemotronVLImageProcessor)
|
||||
|
||||
Reference in New Issue
Block a user