From e5db41b329ef371755c7769bb95fbc881fb953a4 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Oct 2026 16:58:47 -0400 Subject: [PATCH] [Fix] nemotron_vl: Pad visual spans using processor offsets (#44581) Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> --- python/sglang/srt/models/nano_nemotron_vl.py | 11 +++-- .../unit/models/test_nano_nemotron_vl.py | 9 +++- .../test_nano_nemotron_vl_processor.py | 41 +++++++++++++++++++ 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/models/nano_nemotron_vl.py b/python/sglang/srt/models/nano_nemotron_vl.py index c5a5b86495..1f1fc59db3 100644 --- a/python/sglang/srt/models/nano_nemotron_vl.py +++ b/python/sglang/srt/models/nano_nemotron_vl.py @@ -30,6 +30,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.managers.mm_utils import ( general_mm_embed_routine, pad_mm_token_pairs, + pad_mm_tokens, ) from sglang.srt.managers.schedule_batch import ( Modality, @@ -139,20 +140,18 @@ class NemotronH_Nano_VL_V2(EVS): ) def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array: - im_start_id: int = mm_inputs.im_start_id - im_end_id: int = mm_inputs.im_end_id - visual_items = [item for item in mm_inputs.mm_items if not item.is_audio()] audio_items = [item for item in mm_inputs.mm_items if item.is_audio()] all_data_offsets = [] if visual_items: + # Literal boundary tokens in text must not change which spans are images. mm_inputs.mm_items = visual_items - input_ids = pad_mm_token_pairs( - input_ids, mm_inputs, data_token_pairs=[(im_start_id, im_end_id)] + input_ids = pad_mm_tokens(input_ids, mm_inputs) + all_data_offsets.extend( + start - 1 for item in visual_items for start, _ in item.offsets ) - all_data_offsets.extend(mm_inputs.data_offsets) audio_start_id = getattr(mm_inputs, "audio_start_id", None) audio_end_id = getattr(mm_inputs, "audio_end_id", None) diff --git a/test/registered/unit/models/test_nano_nemotron_vl.py b/test/registered/unit/models/test_nano_nemotron_vl.py index cb7d4545e3..24cab5e26e 100644 --- a/test/registered/unit/models/test_nano_nemotron_vl.py +++ b/test/registered/unit/models/test_nano_nemotron_vl.py @@ -99,15 +99,20 @@ class TestNemotronHOmniVideo(CustomTestCase): def test_evs_keeps_image_pad_values_for_redistribution(self): """EVS placeholder redistribution must not restore raw image tokens next to a video.""" start, end, image_token = 1, 2, 18 - image = MultimodalDataItem(modality=Modality.IMAGE, offsets=[], hash=11) + image = MultimodalDataItem(modality=Modality.IMAGE, offsets=[(2, 2)], hash=11) video = _video_item([2], pre_chunked_input_ids=[]) + video.offsets = [(5, 5)] video.hash = 22 for item in (image, video): item.set_pad_value() input_ids = array("q", [7, start, image_token, end, start, image_token, end]) video.pre_chunked_input_ids = input_ids.tolist() mm_inputs = MultimodalInputs( - mm_items=[image, video], im_start_id=start, im_end_id=end + mm_items=[image, video], + im_start_id=start, + im_end_id=end, + im_token_id=image_token, + video_token_id=image_token, ) model = object.__new__(NemotronH_Omni_Reasoning_V3) diff --git a/test/registered/unit/multimodal/test_nano_nemotron_vl_processor.py b/test/registered/unit/multimodal/test_nano_nemotron_vl_processor.py index 4d5a0a3993..4d6ffd0f9f 100644 --- a/test/registered/unit/multimodal/test_nano_nemotron_vl_processor.py +++ b/test/registered/unit/multimodal/test_nano_nemotron_vl_processor.py @@ -2,11 +2,14 @@ import asyncio import unittest +from array import array from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, patch import torch +from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem +from sglang.srt.models.nano_nemotron_vl import NemotronH_Nano_VL_V2 from sglang.srt.multimodal.evs import EVSConfig, EVSProcessor from sglang.srt.multimodal.processors.nano_nemotron_vl import ( NanoNemotronVLImageProcessor, @@ -135,6 +138,44 @@ class TestNanoNemotronVLProcessor(CustomTestCase): ) processor.load_mm_data.assert_not_awaited() + def test_visual_offsets_ignore_literal_boundaries_and_preserve_audio(self): + model = object.__new__(NemotronH_Nano_VL_V2) + for text, start in ( + ([5, 19, 18, 18, 20, 7, 20], 2), + ([5, 19, 9, 19, 18, 18, 20, 7], 4), + ([19, 4, 4, 20, 5, 19, 18, 18, 20], 6), + ): + with self.subTest(text=text): + mm = SimpleNamespace( + mm_items=[ + MultimodalDataItem( + modality=Modality.IMAGE, + offsets=[(start, start + 1)], + pad_value=1_000_001, + ), + MultimodalDataItem( + modality=Modality.AUDIO, + offsets=[(len(text) + 1, len(text) + 2)], + pad_value=1_000_002, + ), + ], + data_offsets=[], + im_start_id=19, + im_end_id=20, + im_token_id=18, + video_token_id=18, + audio_token_id=32, + audio_start_id=30, + audio_end_id=31, + ) + padded = model.pad_input_ids(array("q", text + [30, 32, 32, 31]), mm) + expected = text[:] + expected[start : start + 2] = [1_000_001] * 2 + self.assertEqual( + list(padded), expected + [30, 1_000_002, 1_000_002, 31] + ) + self.assertEqual(mm.data_offsets, [start - 1, len(text)]) + def test_tubelet_label_matches_hf_reference(self): """Multi-frame tubelet labels must match the HF processor's prompt format.""" processor = object.__new__(NanoNemotronVLImageProcessor)