From c2b3e42ad64521c018443f7efe294658a0b1da3b Mon Sep 17 00:00:00 2001 From: Aurick Qiao Date: Thu, 26 Mar 2026 17:38:23 -0700 Subject: [PATCH] Fix sessions with mm inputs (#21269) --- .../scheduler_output_processor_mixin.py | 2 +- .../sglang/srt/managers/session_controller.py | 6 +++ .../sessions/test_session_control.py | 48 ++++++++----------- 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_output_processor_mixin.py b/python/sglang/srt/managers/scheduler_output_processor_mixin.py index 4c759a718..8b6864427 100644 --- a/python/sglang/srt/managers/scheduler_output_processor_mixin.py +++ b/python/sglang/srt/managers/scheduler_output_processor_mixin.py @@ -437,7 +437,7 @@ class SchedulerOutputProcessorMixin: if req.finished(): # delete feature to save memory - if req.multimodal_inputs is not None: + if req.multimodal_inputs is not None and req.session is None: for mm_item in req.multimodal_inputs.mm_items: pixel_values = mm_item.feature if isinstance(pixel_values, torch.Tensor): diff --git a/python/sglang/srt/managers/session_controller.py b/python/sglang/srt/managers/session_controller.py index 05cd15cfa..6b9103027 100644 --- a/python/sglang/srt/managers/session_controller.py +++ b/python/sglang/srt/managers/session_controller.py @@ -164,6 +164,12 @@ class Session: and req.input_ids[0] == tokenizer.bos_token_id ): req.input_ids = req.input_ids[1:] + # Adjust mm_item offsets since they were computed on + # the pre-strip sequence (with BOS at position 0) + if req.mm_inputs: + for item in req.mm_inputs.get("mm_items", []): + if item.offsets: + item.offsets = [(s - 1, e - 1) for s, e in item.offsets] input_ids = ( last_req.origin_input_ids diff --git a/test/registered/sessions/test_session_control.py b/test/registered/sessions/test_session_control.py index 12dbad41b..48dd5f434 100644 --- a/test/registered/sessions/test_session_control.py +++ b/test/registered/sessions/test_session_control.py @@ -571,17 +571,15 @@ class TestSessionControl(unittest.TestCase): ) -@unittest.skip("broken") class TestSessionControlVision(CustomTestCase): @classmethod def setUpClass(cls): - cls.model = "lmms-lab/llava-onevision-qwen2-7b-ov" + cls.model = "OpenGVLab/InternVL2-2B" cls.base_url = DEFAULT_URL_FOR_TEST cls.process = popen_launch_server( cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - # other_args={"--disable-radix"}, ) @classmethod @@ -589,12 +587,13 @@ class TestSessionControlVision(CustomTestCase): kill_process_tree(cls.process.pid) def test_session_control(self): + image_token = "" text_chunks = [ "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n", - "<|im_start|>user\n\nDescribe this image in a very short sentence.<|im_end|>\n<|im_start|>assistant\n", - "<|im_start|>user\n\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n", - "<|im_start|>user\n\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n", - "<|im_start|>user\nDescribe this image in a very short sentence.<|im_end|>\nassistant:", + f"<|im_start|>user\n{image_token}\nDescribe this image in a very short sentence.<|im_end|>\n<|im_start|>assistant\n", + f"<|im_start|>user\n{image_token}\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n", + f"<|im_start|>user\n{image_token}\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n", + "<|im_start|>user\nDescribe this image in a very short sentence.<|im_end|>\n<|im_start|>assistant\n", ] image_chunks = [ "https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png", @@ -605,11 +604,6 @@ class TestSessionControlVision(CustomTestCase): self.assertEqual( len(text_chunks), len(image_chunks) + 2 ) # the first and the last prompt does not contain images - tokenizer = get_tokenizer(self.model) - text_input_ids = [tokenizer.encode(x) for x in text_chunks] - for i in range(1, len(text_input_ids)): - if text_input_ids[i][0] == tokenizer.bos_token_id: - text_input_ids[i] = text_input_ids[i][1:] gen_len = 32 # 1. using session control @@ -629,11 +623,11 @@ class TestSessionControlVision(CustomTestCase): first_rid = None outputs_from_session = [] - for i in range(len(text_input_ids[:-1])): + for i in range(len(text_chunks[:-1])): response = requests.post( self.base_url + "/generate", json={ - "input_ids": text_input_ids[i], + "text": text_chunks[i], "image_data": image_chunks[i - 1] if i > 0 else None, "modalities": ["multi-images"], "session_params": { @@ -662,7 +656,7 @@ class TestSessionControlVision(CustomTestCase): response = requests.post( self.base_url + "/generate", json={ - "input_ids": text_input_ids[-1], + "text": text_chunks[-1], "session_params": { "id": session_id, "rid": first_rid, @@ -683,7 +677,7 @@ class TestSessionControlVision(CustomTestCase): ret = requests.post( self.base_url + "/generate", json={ - "input_ids": text_input_ids[-1], + "text": text_chunks[-1], "session_params": { "id": session_id, "rid": rid, @@ -710,7 +704,7 @@ class TestSessionControlVision(CustomTestCase): ret = requests.post( self.base_url + "/generate", json={ - "input_ids": text_input_ids[-1], + "text": text_chunks[-1], "session_params": { "id": session_id, "rid": first_rid, @@ -730,16 +724,16 @@ class TestSessionControlVision(CustomTestCase): # 2. not use session control requests.post(self.base_url + "/flush_cache") - input_ids_first_req = None - input_ids = [] + accumulated_text = "" + first_req_text = None outputs_normal = [] - for i in range(len(text_input_ids[:-1])): - input_ids += text_input_ids[i] + for i in range(len(text_chunks[:-1])): + accumulated_text += text_chunks[i] image_data = image_chunks[:i] if i > 0 else None response = requests.post( self.base_url + "/generate", json={ - "input_ids": input_ids, + "text": accumulated_text, "image_data": image_data, "modalities": ["multi-images"], "sampling_params": { @@ -753,19 +747,15 @@ class TestSessionControlVision(CustomTestCase): }, ).json() if i > 0: - output_ids = tokenizer.encode(response["text"]) - if output_ids[0] == tokenizer.bos_token_id: - output_ids = output_ids[1:] - input_ids += output_ids + accumulated_text += response["text"] outputs_normal.append(response["text"]) if i == 0: - input_ids_first_req = input_ids.copy() + first_req_text = accumulated_text - input_ids_first_req += text_input_ids[-1] response = requests.post( self.base_url + "/generate", json={ - "input_ids": input_ids_first_req, + "text": first_req_text + text_chunks[-1], "sampling_params": { "temperature": 0, "max_new_tokens": gen_len,