minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix sessions with mm inputs (#21269)

Browse Source
This commit is contained in:
Aurick Qiao
2026-03-26 17:38:23 -07:00
committed by GitHub
parent 8a4cdcd538
commit c2b3e42ad6
3 changed files with 26 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/scheduler_output_processor_mixin.py
+1 -1
View File
@@ -437,7 +437,7 @@ class SchedulerOutputProcessorMixin:
if req.finished(): if req.finished():
# delete feature to save memory # delete feature to save memory
if req.multimodal_inputs is not None: if req.multimodal_inputs is not None and req.session is None:
for mm_item in req.multimodal_inputs.mm_items: for mm_item in req.multimodal_inputs.mm_items:
pixel_values = mm_item.feature pixel_values = mm_item.feature
if isinstance(pixel_values, torch.Tensor): if isinstance(pixel_values, torch.Tensor):
python/sglang/srt/managers/session_controller.py
+6
View File
@@ -164,6 +164,12 @@ class Session:
and req.input_ids[0] == tokenizer.bos_token_id and req.input_ids[0] == tokenizer.bos_token_id
): ):
req.input_ids = req.input_ids[1:] req.input_ids = req.input_ids[1:]
# Adjust mm_item offsets since they were computed on
# the pre-strip sequence (with BOS at position 0)
if req.mm_inputs:
for item in req.mm_inputs.get("mm_items", []):
if item.offsets:
item.offsets = [(s - 1, e - 1) for s, e in item.offsets]
input_ids = ( input_ids = (
last_req.origin_input_ids last_req.origin_input_ids
test/registered/sessions/test_session_control.py
+19 -29
View File
@@ -571,17 +571,15 @@ class TestSessionControl(unittest.TestCase):
) )
@unittest.skip("broken")
class TestSessionControlVision(CustomTestCase): class TestSessionControlVision(CustomTestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.model = "lmms-lab/llava-onevision-qwen2-7b-ov" cls.model = "OpenGVLab/InternVL2-2B"
cls.base_url = DEFAULT_URL_FOR_TEST cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server( cls.process = popen_launch_server(
cls.model, cls.model,
cls.base_url, cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
# other_args={"--disable-radix"},
) )
@classmethod @classmethod
@@ -589,12 +587,13 @@ class TestSessionControlVision(CustomTestCase):
kill_process_tree(cls.process.pid) kill_process_tree(cls.process.pid)
def test_session_control(self): def test_session_control(self):
image_token = "<IMG_CONTEXT>"
text_chunks = [ text_chunks = [
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n", "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n",
"<|im_start|>user\n<image>\nDescribe this image in a very short sentence.<|im_end|>\n<|im_start|>assistant\n", f"<|im_start|>user\n{image_token}\nDescribe this image in a very short sentence.<|im_end|>\n<|im_start|>assistant\n",
"<|im_start|>user\n<image>\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n", f"<|im_start|>user\n{image_token}\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n",
"<|im_start|>user\n<image>\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n", f"<|im_start|>user\n{image_token}\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n",
"<|im_start|>user\nDescribe this image in a very short sentence.<|im_end|>\nassistant:", "<|im_start|>user\nDescribe this image in a very short sentence.<|im_end|>\n<|im_start|>assistant\n",
] ]
image_chunks = [ image_chunks = [
"https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png", "https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png",
@@ -605,11 +604,6 @@ class TestSessionControlVision(CustomTestCase):
self.assertEqual( self.assertEqual(
len(text_chunks), len(image_chunks) + 2 len(text_chunks), len(image_chunks) + 2
) # the first and the last prompt does not contain images ) # the first and the last prompt does not contain images
tokenizer = get_tokenizer(self.model)
text_input_ids = [tokenizer.encode(x) for x in text_chunks]
for i in range(1, len(text_input_ids)):
if text_input_ids[i][0] == tokenizer.bos_token_id:
text_input_ids[i] = text_input_ids[i][1:]
gen_len = 32 gen_len = 32
# 1. using session control # 1. using session control
@@ -629,11 +623,11 @@ class TestSessionControlVision(CustomTestCase):
first_rid = None first_rid = None
outputs_from_session = [] outputs_from_session = []
for i in range(len(text_input_ids[:-1])): for i in range(len(text_chunks[:-1])):
response = requests.post( response = requests.post(
self.base_url + "/generate", self.base_url + "/generate",
json={ json={
"input_ids": text_input_ids[i], "text": text_chunks[i],
"image_data": image_chunks[i - 1] if i > 0 else None, "image_data": image_chunks[i - 1] if i > 0 else None,
"modalities": ["multi-images"], "modalities": ["multi-images"],
"session_params": { "session_params": {
@@ -662,7 +656,7 @@ class TestSessionControlVision(CustomTestCase):
response = requests.post( response = requests.post(
self.base_url + "/generate", self.base_url + "/generate",
json={ json={
"input_ids": text_input_ids[-1], "text": text_chunks[-1],
"session_params": { "session_params": {
"id": session_id, "id": session_id,
"rid": first_rid, "rid": first_rid,
@@ -683,7 +677,7 @@ class TestSessionControlVision(CustomTestCase):
ret = requests.post( ret = requests.post(
self.base_url + "/generate", self.base_url + "/generate",
json={ json={
"input_ids": text_input_ids[-1], "text": text_chunks[-1],
"session_params": { "session_params": {
"id": session_id, "id": session_id,
"rid": rid, "rid": rid,
@@ -710,7 +704,7 @@ class TestSessionControlVision(CustomTestCase):
ret = requests.post( ret = requests.post(
self.base_url + "/generate", self.base_url + "/generate",
json={ json={
"input_ids": text_input_ids[-1], "text": text_chunks[-1],
"session_params": { "session_params": {
"id": session_id, "id": session_id,
"rid": first_rid, "rid": first_rid,
@@ -730,16 +724,16 @@ class TestSessionControlVision(CustomTestCase):
# 2. not use session control # 2. not use session control
requests.post(self.base_url + "/flush_cache") requests.post(self.base_url + "/flush_cache")
input_ids_first_req = None accumulated_text = ""
input_ids = [] first_req_text = None
outputs_normal = [] outputs_normal = []
for i in range(len(text_input_ids[:-1])): for i in range(len(text_chunks[:-1])):
input_ids += text_input_ids[i] accumulated_text += text_chunks[i]
image_data = image_chunks[:i] if i > 0 else None image_data = image_chunks[:i] if i > 0 else None
response = requests.post( response = requests.post(
self.base_url + "/generate", self.base_url + "/generate",
json={ json={
"input_ids": input_ids, "text": accumulated_text,
"image_data": image_data, "image_data": image_data,
"modalities": ["multi-images"], "modalities": ["multi-images"],
"sampling_params": { "sampling_params": {
@@ -753,19 +747,15 @@ class TestSessionControlVision(CustomTestCase):
}, },
).json() ).json()
if i > 0: if i > 0:
output_ids = tokenizer.encode(response["text"]) accumulated_text += response["text"]
if output_ids[0] == tokenizer.bos_token_id:
output_ids = output_ids[1:]
input_ids += output_ids
outputs_normal.append(response["text"]) outputs_normal.append(response["text"])
if i == 0: if i == 0:
input_ids_first_req = input_ids.copy() first_req_text = accumulated_text
input_ids_first_req += text_input_ids[-1]
response = requests.post( response = requests.post(
self.base_url + "/generate", self.base_url + "/generate",
json={ json={
"input_ids": input_ids_first_req, "text": first_req_text + text_chunks[-1],
"sampling_params": { "sampling_params": {
"temperature": 0, "temperature": 0,
"max_new_tokens": gen_len, "max_new_tokens": gen_len,