minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[Perf] Optimize Qwen3-VL unique-image serving on H100 (#36411)

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Xiaoyu Zhang
2026-09-12 15:16:59 +08:00
committed by GitHub
co-authored by Cursor
parent 6ba96d329f
commit 0a57403468
30 changed files with 1370 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/chunked_prefill/test_mm_chunked_embedding_unit.py
+29
View File
@@ -17,6 +17,11 @@ import torch
from sglang.srt.managers import mm_schedule
from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
from sglang.srt.multimodal.transport.cuda_ipc import (
BORROW_CUDA_IPC_FEATURE_KEY,
DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY,
CudaIpcTensorTransportProxy,
)
from sglang.srt.runtime_context import get_context, get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
@@ -235,6 +240,30 @@ def test_batched_mismatched_cache_entry_is_reencoded():
encoder.assert_called_once()
def test_full_deferred_ipc_item_is_marked_for_borrow():
mm_schedule.init_mm_embedding_cache(1 << 30)
proxy = object.__new__(CudaIpcTensorTransportProxy)
item = MultimodalDataItem(
modality=Modality.IMAGE,
hash=1000,
pad_value=1000,
feature=proxy,
offsets=[ITEM_OFFSETS[0]],
model_specific_data={DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY: True},
)
request = mm_schedule.PerImageRequestInfo(
req_idx=0,
items=[item],
items_offset=[ITEM_OFFSETS[0]],
extend_prefix_len=0,
extend_seq_len=TOTAL_LEN,
)
mm_schedule._batch_encode_per_image_misses(_encoder_list, [request], _CPU)
assert item.model_specific_data[BORROW_CUDA_IPC_FEATURE_KEY]
def test_batched_colliding_hashes_with_different_lengths_are_not_deduplicated():
mm_schedule.init_mm_embedding_cache(1 << 30)
items = _make_items()
test/registered/unit/entrypoints/openai/test_serving_chat.py
+77 -4
View File
@@ -6,7 +6,7 @@ or
python -m unittest discover -s tests -p "test_*unit.py" -v
"""
from sglang.test.test_utils import enter_override, maybe_stub_sgl_kernel
from sglang.test.test_utils import CustomTestCase, enter_override, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
@@ -195,6 +195,81 @@ class _MockTemplateManager:
self.jinja_template_may_reorder_tool_results = False
class TestChatTemplateCache(CustomTestCase):
def setUp(self):
super().setUp()
reset_context()
self.addCleanup(reset_context)
publish(
ServerArgs(model_path="dummy", default_chat_template_kwargs=None),
role="tokenizer",
)
self.tokenizer_manager = _MockTokenizerManager()
self.chat = OpenAIServingChat(
self.tokenizer_manager,
_MockTemplateManager(),
)
self.tokenizer_manager.tokenizer.apply_chat_template.return_value = "rendered"
self.tokenizer_manager.tokenizer.encode.return_value = [11, 12]
self.tokenizer_manager.tokenizer.decode.return_value = "decoded"
self.tokenizer_manager.tokenizer.reset_mock()
def _render(self, **overrides):
kwargs = {
"messages": [{"role": "user", "content": "same text prefix"}],
"tools": None,
"template_kwargs": {"enable_thinking": False},
"encode_kwargs": {"add_special_tokens": False},
"use_cache": True,
}
kwargs.update(overrides)
return self.chat._render_and_encode_chat_template(**kwargs)
def test_cache_hit_reuses_render_encode_and_returns_an_owned_id_list(self):
first = self._render()
first[1].append(99)
second = self._render()
self.assertEqual(second, ("rendered", [11, 12], "decoded"))
self.tokenizer_manager.tokenizer.apply_chat_template.assert_called_once()
self.tokenizer_manager.tokenizer.encode.assert_called_once()
self.tokenizer_manager.tokenizer.decode.assert_called_once()
def test_cache_key_includes_template_and_encode_options(self):
self._render()
self._render(template_kwargs={"enable_thinking": True})
self._render(encode_kwargs={"add_special_tokens": True})
self.assertEqual(
self.tokenizer_manager.tokenizer.apply_chat_template.call_count,
3,
)
self.assertEqual(self.tokenizer_manager.tokenizer.encode.call_count, 3)
def test_cache_key_tracks_tokenizer_chat_template_updates(self):
self.tokenizer_manager.tokenizer.chat_template = "template-v1"
self._render()
self.tokenizer_manager.tokenizer.chat_template = "template-v2"
self._render()
self.assertEqual(
self.tokenizer_manager.tokenizer.apply_chat_template.call_count,
2,
)
def test_non_serializable_input_bypasses_cache(self):
messages = [{"role": "user", "content": object()}]
self._render(messages=messages)
self._render(messages=messages)
self.assertEqual(
self.tokenizer_manager.tokenizer.apply_chat_template.call_count,
2,
)
self.assertEqual(self.tokenizer_manager.tokenizer.encode.call_count, 2)
self.tokenizer_manager.tokenizer.decode.assert_not_called()
class ServingChatTestCase(unittest.TestCase):
# ------------- common fixtures -------------
def setUp(self):
@@ -386,9 +461,7 @@ class ServingChatTestCase(unittest.TestCase):
)
self.tm.tokenizer.apply_chat_template.reset_mock()
self.chat._apply_jinja_template(ordered_request, None, is_multimodal=True)
self.assertEqual(
rendered_messages, self.tm.tokenizer.apply_chat_template.call_args[0][0]
)
self.tm.tokenizer.apply_chat_template.assert_not_called()
self.template_manager.jinja_template_may_reorder_tool_results = False
self.tm.tokenizer.apply_chat_template.reset_mock()
test/registered/unit/mem_cache/test_radix_cache_unit.py
+48 -1
View File
@@ -50,6 +50,7 @@ from sglang.srt.mem_cache.events import KVCacheEventRecorder
from sglang.srt.mem_cache.mamba_radix_cache import TreeNode as MambaTreeNode
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
from sglang.srt.utils import get_device
from sglang.test.test_utils import CustomTestCase
# Test constants
DEFAULT_PAGE_SIZE = 4
@@ -368,7 +369,7 @@ class TestTreeNode(unittest.TestCase):
self.assertEqual(n4.get_prefix_hash_values(n3), ["h1", "h2", "h3"])
class TestRadixCache(unittest.TestCase):
class TestRadixCache(CustomTestCase):
"""Test cases for RadixCache class."""
def setUp(self):
@@ -540,6 +541,52 @@ class TestRadixCache(unittest.TestCase):
cache.req_to_token_pool.req_to_token[0], tree_indices
)
def test_finished_request_splits_prompt_from_output_for_eviction(self):
class ReqToTokenPool:
def __init__(self, row):
self.req_to_token = row.unsqueeze(0)
allocator = TokenToKVPoolAllocator(
size=16,
dtype=torch.float16,
device="cpu",
kvcache=None,
need_sort=False,
)
cache = RadixCache.create_simulated(mock_allocator=allocator)
prompt_ids = array("q", [1, 2, 3])
output_ids = array("q", [4, 5])
kv_indices = allocator.alloc(len(prompt_ids) + len(output_ids))
self.assertIsNotNone(kv_indices)
cache.req_to_token_pool = ReqToTokenPool(kv_indices)
req = unittest.mock.Mock(
origin_input_ids=prompt_ids,
output_ids=output_ids,
kv=ReqKvInfo(req_pool_idx=0, cache_protected_len=0),
extra_key=None,
cache_salt=None,
priority=0,
last_node=cache.root_node,
)
cache.cache_finished_req(
req,
is_insert=True,
kv_len_to_handle=len(prompt_ids) + len(output_ids),
)
(prompt_node,) = cache.root_node.children.values()
(output_node,) = prompt_node.children.values()
self.assertEqual(len(prompt_node.key), len(prompt_ids))
self.assertEqual(len(output_node.key), len(output_ids))
result = cache.evict(EvictParams(num_tokens=len(output_ids)))
self.assertEqual(result.num_tokens_evicted, len(output_ids))
match = cache.match_prefix(
MatchPrefixParams(key=RadixKey(prompt_ids + output_ids))
)
self.assertEqual(len(match.device_indices), len(prompt_ids))
def test_kv_cache_events(self):
"""Test KV cache events functionality."""
test_cases = [
test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py
+22
View File
@@ -1558,6 +1558,28 @@ class UnifiedRadixCacheSuite:
MatchPrefixParams(key=RadixKey(array("q", all_ids[:aligned_len])))
)
self.assertEqual(len(m.device_indices), aligned_len)
prompt_aligned_len = (len(input_ids) // ps) * ps
if self.cfg.components == (ComponentType.FULL,):
(prompt_node,) = _node_children(cache, cache.root_node_handle())
(output_node,) = _node_children(cache, prompt_node)
self.assertEqual(_node_key_length(cache, prompt_node), prompt_aligned_len)
self.assertEqual(
_node_key_length(cache, output_node),
aligned_len - prompt_aligned_len,
)
result = cache.evict(
EvictParams(num_tokens=aligned_len - prompt_aligned_len)
)
self.assertEqual(
result.num_tokens_evicted,
aligned_len - prompt_aligned_len,
)
prompt_only = cache.match_prefix(
MatchPrefixParams(key=RadixKey(array("q", all_ids[:aligned_len])))
)
self.assertEqual(len(prompt_only.device_indices), prompt_aligned_len)
cache.sanity_check()
def test_cache_finished_req_strips_thinking(self):
test/registered/unit/models/test_qwen3_vl_feature_materialization.py
+101 -8
View File
@@ -1,16 +1,24 @@
"""Regression tests for Qwen3-VL multimodal feature materialization."""
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputs,
)
from sglang.srt.models.qwen3_vl import Qwen3VLForConditionalGeneration
from sglang.srt.multimodal.processors.qwen_vl import QwenVLImageProcessor
from sglang.srt.multimodal.transport.cuda_ipc import (
BORROW_CUDA_IPC_FEATURE_KEY,
CUDA_IPC_FEATURE_COPY_EVENT_KEY,
DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY,
CudaIpcTensorTransportProxy,
)
from sglang.srt.runtime_context import get_context
from sglang.test.ci.ci_register import register_cpu_ci
@@ -69,7 +77,7 @@ class TestQwen3VLFeatureMaterialization(CustomTestCase):
MultimodalDataItem(modality=Modality.AUDIO),
]
processor._mark_dp_encoder_features_for_deferred_reconstruction(items)
processor._mark_cuda_ipc_features_for_deferred_reconstruction(items)
self.assertTrue(
items[0].model_specific_data[
@@ -87,19 +95,104 @@ class TestQwen3VLFeatureMaterialization(CustomTestCase):
)
def test_processor_does_not_defer_cpu_transport(self):
processor = QwenVLImageProcessor.__new__(QwenVLImageProcessor)
processor.mm_feature_transport = "cpu"
processor.server_args = SimpleNamespace(mm_enable_dp_encoder=True)
processor.model_type = "qwen3_vl"
item = MultimodalDataItem(modality=Modality.IMAGE)
with get_context().override_server_args(mm_enable_dp_encoder=True):
processor = QwenVLImageProcessor.__new__(QwenVLImageProcessor)
processor.mm_feature_transport = "cpu"
processor.model_type = "qwen3_vl"
item = MultimodalDataItem(modality=Modality.IMAGE)
processor._mark_dp_encoder_features_for_deferred_reconstruction([item])
processor._mark_cuda_ipc_features_for_deferred_reconstruction([item])
self.assertNotIn(
DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY,
item.model_specific_data,
)
def test_processor_defers_cuda_ipc_for_single_tp_qwen3_vl(self):
processor = QwenVLImageProcessor.__new__(QwenVLImageProcessor)
processor.mm_feature_transport = "cuda_ipc"
processor.model_type = "qwen3_vl"
item = MultimodalDataItem(modality=Modality.IMAGE)
processor._mark_cuda_ipc_features_for_deferred_reconstruction([item])
self.assertTrue(
item.model_specific_data[DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY]
)
def test_retract_reprefill_waits_for_preserved_visual_input(self):
visual = Mock()
visual.device = torch.device("cuda:0")
visual.dtype = torch.bfloat16
visual.side_effect = lambda pixel_values, *, grid_thw: pixel_values
model = self._model(visual, use_data_parallel=False)
proxy = CudaIpcTensorTransportProxy.__new__(CudaIpcTensorTransportProxy)
proxy.total_consumer_count = 1
borrowed_feature = torch.ones(2, 3)
packed_ready = Mock()
host_ready = Mock()
current_stream = Mock()
copy_stream = Mock()
proxy.reconstruct_on_target_device = Mock()
proxy.borrow_on_target_device = Mock(return_value=borrowed_feature)
proxy.release_borrowed_on_current_stream = Mock()
proxy.release_without_reconstruction = Mock()
item = MultimodalDataItem(
modality=Modality.IMAGE,
feature=proxy,
model_specific_data={BORROW_CUDA_IPC_FEATURE_KEY: True},
)
item.image_grid_thw = torch.tensor([[1, 1, 2]])
with (
patch(
"sglang.srt.models.qwen3_vl.get_parallel",
return_value=SimpleNamespace(tp_size=1),
),
patch(
"sglang.srt.models.qwen3_vl.materialize_multimodal_features",
side_effect=lambda features, **_kwargs: torch.cat(features),
),
patch(
"sglang.srt.models.qwen3_vl.torch.cuda.current_stream",
return_value=current_stream,
),
patch(
"sglang.srt.models.qwen3_vl.torch.cuda.Event",
side_effect=(packed_ready, host_ready),
),
patch(
"sglang.srt.models.qwen3_vl.torch.cuda.Stream",
return_value=copy_stream,
),
patch(
"sglang.srt.models.qwen3_vl.torch.cuda.stream",
return_value=nullcontext(),
),
):
first = model.get_image_feature([item])
second = model.get_image_feature([item])
self.assertIsNot(item.feature, borrowed_feature)
self.assertTrue(torch.equal(item.feature, borrowed_feature))
self.assertTrue(torch.equal(first, second))
proxy.reconstruct_on_target_device.assert_not_called()
proxy.borrow_on_target_device.assert_called_once_with(0)
proxy.release_borrowed_on_current_stream.assert_called_once_with()
proxy.release_without_reconstruction.assert_not_called()
packed_ready.record.assert_called_once_with(current_stream)
copy_stream.wait_event.assert_called_once_with(packed_ready)
host_ready.record.assert_called_once_with(copy_stream)
current_stream.wait_event.assert_called_once_with(host_ready)
self.assertEqual(visual.call_count, 2)
MultimodalInputs(mm_items=[item]).release_features()
proxy.release_without_reconstruction.assert_not_called()
self.assertIsNone(item.feature)
self.assertNotIn(CUDA_IPC_FEATURE_COPY_EVENT_KEY, item.model_specific_data)
def test_image_features_are_packed_on_the_visual_device(self):
visual = _RecordingVisual()
model = self._model(visual, use_data_parallel=False)
test/registered/unit/multimodal/rust/qwen/_fixtures.py
+1
View File
@@ -71,6 +71,7 @@ def make_processor(case, config, image_processor_cls=None):
model_impl="sglang",
keep_mm_feature_on_device=False,
mm_feature_transport="cpu",
mm_enable_dp_encoder=False,
image_processor_backend="auto",
disable_fast_image_processor=True,
skip_tokenizer_init=False,
test/registered/unit/multimodal/test_cuda_ipc_transport.py
+55
View File
@@ -128,6 +128,61 @@ class TestCudaIpcTransport(CustomTestCase):
producer.join(timeout=10)
self.assertEqual(producer.exitcode, 0)
def test_borrowed_tensor_keeps_lease_until_explicit_release(self):
ctx = mp.get_context("spawn")
proxy_queue = ctx.Queue()
producer_results = ctx.Queue()
consumer_done = ctx.Event()
producer = ctx.Process(
target=_produce_pooled_tensor,
args=(proxy_queue, consumer_done, producer_results),
)
producer.start()
proxy = borrowed = consumed = None
producer_result = None
try:
try:
proxy, expected = proxy_queue.get(timeout=60)
except queue.Empty:
producer_result = producer_results.get(timeout=5)
_status, payload = producer_result
self.fail(
f"CUDA IPC producer failed before sending its proxy: {payload}"
)
borrowed = proxy.borrow_on_target_device(0)
self.assertIsNotNone(borrowed)
consumed = borrowed + 1
torch.cuda.synchronize()
self.assertFalse(proxy._consumer_acknowledged)
proxy.release_without_reconstruction()
torch.cuda.synchronize()
self.assertEqual(
consumed.cpu().tolist(),
(torch.tensor(expected) + 1).tolist(),
)
self.assertTrue(proxy._consumer_acknowledged)
self.assertIsNone(proxy._borrowed_storage)
finally:
del consumed, borrowed, proxy
_pool_handle_cache_clear()
gc.collect()
torch.cuda.ipc_collect()
consumer_done.set()
producer.join(timeout=60)
try:
if producer_result is None:
producer_result = producer_results.get(timeout=5)
status, payload = producer_result
self.assertEqual(status, "ok", payload)
finally:
if producer.is_alive():
producer.terminate()
producer.join(timeout=10)
self.assertEqual(producer.exitcode, 0)
def test_failed_reconstruction_releases_pooled_tensor(self):
ctx = mp.get_context("spawn")
proxy_queue = ctx.Queue()
test/registered/unit/multimodal/test_media_artifact_processor.py
+35 -1
View File
@@ -15,6 +15,7 @@ from sglang.srt.multimodal.media_artifacts import (
MediaArtifactInput,
)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
@@ -88,7 +89,7 @@ class _Processor(MediaArtifactCacheMixin):
self.io_executor.shutdown()
class TestMediaArtifactProcessor(unittest.TestCase):
class TestMediaArtifactProcessor(CustomTestCase):
def test_default_image_decoder_rejects_lazy_pil_failure(self):
malformed_png = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIHWP4z8DwHwAFgAI/ScLJSwAAAABJRU5ErkJggg=="
@@ -253,6 +254,39 @@ class TestMediaArtifactProcessor(unittest.TestCase):
self.assertEqual(artifacts[0].feature, b"fresh")
self.assertEqual(len(processor.batches), 1)
def test_without_cache_builds_request_local_artifacts_and_does_not_retain(self):
processor = _Processor()
digest = snapshot_media(b"image").content_digest
key = processor._artifact_key(digest, b"image")
cached = _Artifact(digest, key, 1, b"cached")
processor.mm_preprocess_cache.put(key, cached)
try:
artifacts = asyncio.run(
processor.prepare_media_artifacts_without_cache([b"image", b"image"])
)
cached_after = asyncio.run(processor.prepare_media_artifacts([b"image"]))
finally:
processor.close()
self.assertEqual([artifact.feature for artifact in artifacts], [b"image"] * 2)
self.assertEqual(len(processor.batches), 1)
self.assertEqual(len(processor.batches[0]), 2)
self.assertIs(cached_after[0], cached)
def test_without_cache_still_validates_caller_content_hash(self):
processor = _Processor()
try:
with self.assertRaisesRegex(ValueError, "content hash mismatch"):
asyncio.run(
processor.prepare_media_artifacts_without_cache(
[b"image"],
content_hashes=[snapshot_media(b"different").content_digest],
)
)
finally:
processor.close()
if __name__ == "__main__":
unittest.main()
test/registered/unit/server_args/test_server_args.py
+19
View File
@@ -111,6 +111,25 @@ _mock_device.start()
class TestPrepareServerArgs(CustomTestCase):
def test_radix_eviction_policy_explicitness_is_preserved(self):
omitted = prepare_server_args(["--model-path", "dummy"])
separated = prepare_server_args(
["--model-path", "dummy", "--radix-eviction-policy", "lru"]
)
joined = prepare_server_args(
["--model-path", "dummy", "--radix-eviction-policy=lru"]
)
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
f.write("model-path: dummy\nradix-eviction-policy: lru\n")
config_path = f.name
self.addCleanup(os.unlink, config_path)
configured = prepare_server_args(["--config", config_path])
self.assertFalse(omitted._radix_eviction_policy_explicitly_set)
self.assertTrue(separated._radix_eviction_policy_explicitly_set)
self.assertTrue(joined._radix_eviction_policy_explicitly_set)
self.assertTrue(configured._radix_eviction_policy_explicitly_set)
def test_ple_embedding_offload_rejects_generic_weight_offload(self):
for generic_offload in (
{"cpu_offload_gb": 1},
test/registered/unit/test_model_overrides.py
+107
View File
@@ -20,6 +20,7 @@ from sglang.srt.arg_groups import overrides as overrides_module
from sglang.srt.arg_groups.arg_utils import A, Arg, resolvable_fields
from sglang.srt.arg_groups.model_overrides import minicpm as minicpm_module
from sglang.srt.arg_groups.model_overrides import qwen3_5 as qwen3_5_module
from sglang.srt.arg_groups.model_overrides import qwen3_vl as qwen3_vl_module
from sglang.srt.arg_groups.overrides import (
collect_model_override_declarations,
register_model_override,
@@ -108,6 +109,10 @@ class TestModelOverridableWhitelist(CustomTestCase):
"enable_symm_mem",
"speculative_attention_mode",
"speculative_draft_attention_backend",
"prefill_decode_interval",
"radix_eviction_policy",
"mm_preprocess_cache_size_mb",
"mm_feature_transport",
}
),
)
@@ -3107,5 +3112,107 @@ class TestDeclarationValidation(CustomTestCase):
validate_declarations(args, [("src", {"nope": 1})])
class TestQwen3VLHopperServingOverrides(CustomTestCase):
def setUp(self):
super().setUp()
self.addCleanup(envs.SGLANG_VLM_CACHE_SIZE_MB.clear)
self.addCleanup(envs.SGLANG_MM_FEATURE_CACHE_MB.clear)
envs.SGLANG_VLM_CACHE_SIZE_MB.clear()
envs.SGLANG_MM_FEATURE_CACHE_MB.clear()
@staticmethod
def _args(**overrides):
from sglang.srt.server_args import ServerArgs
values = {
"mm_preprocess_cache_size_mb": None,
"mm_feature_transport": None,
"max_running_requests": 400,
"radix_eviction_policy": "lru",
"prefill_decode_interval": None,
"attention_backend": None,
"decode_attention_backend": None,
}
values.update(overrides)
return ServerArgs(model_path="dummy", **values)
@patch.object(
qwen3_vl_module,
"large_hopper_qwen3_vl_model_type",
return_value="qwen3_vl",
)
def test_profiled_defaults_are_valid_model_overrides(self, _mock_model_type):
server_args = self._args()
updates = qwen3_vl_module._qwen3vl_hopper_serving_overrides(server_args, None)
self.assertEqual(
updates,
{
"mm_preprocess_cache_size_mb": 0,
"mm_feature_transport": "cuda_ipc",
"radix_eviction_policy": "priority",
"prefill_decode_interval": 22,
"decode_attention_backend": "flashinfer",
},
)
validate_declarations(
server_args,
[("_qwen3vl_hopper_serving_overrides", updates)],
)
self.assertEqual(envs.SGLANG_VLM_CACHE_SIZE_MB.get(), 0)
self.assertEqual(envs.SGLANG_MM_FEATURE_CACHE_MB.get(), 3 * 1024)
@patch.object(
qwen3_vl_module,
"large_hopper_qwen3_vl_model_type",
return_value="qwen3_vl",
)
def test_multinode_does_not_auto_select_cuda_ipc(self, _mock_model_type):
updates = qwen3_vl_module._qwen3vl_hopper_serving_overrides(
self._args(nnodes=2), None
)
self.assertNotIn("mm_feature_transport", updates)
self.assertFalse(envs.SGLANG_MM_FEATURE_CACHE_MB.is_set())
@patch.object(
qwen3_vl_module,
"large_hopper_qwen3_vl_model_type",
side_effect=AssertionError("must not load model config without GPU memory"),
)
def test_decode_graph_expansion_skips_unknown_gpu_memory(self, _mock_model_type):
decode_config = SimpleNamespace(max_bs=256)
qwen3_vl_module.expand_multimodal_decode_graph_to_running_limit(
self._args(), decode_config, gpu_mem=None
)
self.assertEqual(decode_config.max_bs, 256)
@patch.object(
qwen3_vl_module,
"large_hopper_qwen3_vl_model_type",
return_value="qwen3_vl",
)
def test_explicit_choices_are_not_replaced(self, _mock_model_type):
envs.SGLANG_VLM_CACHE_SIZE_MB.set(512)
envs.SGLANG_MM_FEATURE_CACHE_MB.set(2048)
updates = qwen3_vl_module._qwen3vl_hopper_serving_overrides(
self._args(
mm_preprocess_cache_size_mb=256,
mm_feature_transport="cpu",
radix_eviction_policy="lru",
_radix_eviction_policy_explicitly_set=True,
prefill_decode_interval=0,
decode_attention_backend="fa3",
),
None,
)
self.assertEqual(updates, {})
self.assertEqual(envs.SGLANG_VLM_CACHE_SIZE_MB.get(), 512)
self.assertEqual(envs.SGLANG_MM_FEATURE_CACHE_MB.get(), 2048)
if __name__ == "__main__":
unittest.main()