diff --git a/test/registered/eval/test_vlms_mmmu_eval.py b/test/registered/eval/test_vlms_mmmu_eval.py index d8448e3ff..7eb2c1498 100644 --- a/test/registered/eval/test_vlms_mmmu_eval.py +++ b/test/registered/eval/test_vlms_mmmu_eval.py @@ -44,8 +44,9 @@ MODEL_THRESHOLDS = { ModelLaunchSettings("moonshotai/Kimi-VL-A3B-Instruct"): ModelEvalMetrics( 0.330, 23.5 ), - ModelLaunchSettings("openbmb/MiniCPM-o-2_6"): ModelEvalMetrics(0.330, 29.5), - ModelLaunchSettings("openbmb/MiniCPM-v-2_6"): ModelEvalMetrics(0.259, 36.3), + # temporarily disabled: NaN in next_token_logits + # ModelLaunchSettings("openbmb/MiniCPM-o-2_6"): ModelEvalMetrics(0.330, 29.5), + # ModelLaunchSettings("openbmb/MiniCPM-v-2_6"): ModelEvalMetrics(0.259, 36.3), ModelLaunchSettings("OpenGVLab/InternVL2_5-2B"): ModelEvalMetrics(0.300, 18.0), ModelLaunchSettings("Qwen/Qwen2-VL-7B-Instruct"): ModelEvalMetrics(0.310, 83.3), ModelLaunchSettings("Qwen/Qwen2.5-VL-7B-Instruct"): ModelEvalMetrics(0.330, 31.9), diff --git a/test/registered/models/test_vlm_models.py b/test/registered/models/test_vlm_models.py index a0eea75ae..9e49f33ac 100644 --- a/test/registered/models/test_vlm_models.py +++ b/test/registered/models/test_vlm_models.py @@ -19,12 +19,15 @@ register_amd_ci(est_time=850, suite="stage-b-test-1-gpu-small-amd-nondeterminist _is_hip = is_hip() # VLM models for testing if _is_hip: - MODELS = [SimpleNamespace(model="openbmb/MiniCPM-V-2_6", mmmu_accuracy=0.4)] + MODELS = [ + # SimpleNamespace(model="openbmb/MiniCPM-V-2_6", mmmu_accuracy=0.4), # temporarily disabled: NaN in next_token_logits + SimpleNamespace(model="Qwen/Qwen2.5-VL-3B-Instruct", mmmu_accuracy=0.4), + ] else: MODELS = [ SimpleNamespace(model="google/gemma-3-4b-it", mmmu_accuracy=0.38), SimpleNamespace(model="Qwen/Qwen2.5-VL-3B-Instruct", mmmu_accuracy=0.4), - SimpleNamespace(model="openbmb/MiniCPM-V-2_6", mmmu_accuracy=0.4), + # SimpleNamespace(model="openbmb/MiniCPM-V-2_6", mmmu_accuracy=0.4), # temporarily disabled: NaN in next_token_logits ] diff --git a/test/registered/vlm/test_vision_openai_server_a.py b/test/registered/vlm/test_vision_openai_server_a.py index 95814220e..85cb642e3 100644 --- a/test/registered/vlm/test_vision_openai_server_a.py +++ b/test/registered/vlm/test_vision_openai_server_a.py @@ -120,6 +120,7 @@ class TestInternVL25Server(ImageOpenAITestMixin): ] +@unittest.skip("temporarily disabled: NaN in next_token_logits") class TestMiniCPMV4Server(ImageOpenAITestMixin): model = "openbmb/MiniCPM-V-4" extra_args = [ @@ -127,6 +128,7 @@ class TestMiniCPMV4Server(ImageOpenAITestMixin): ] +@unittest.skip("temporarily disabled: NaN in next_token_logits") class TestMiniCPMo26Server(ImageOpenAITestMixin, AudioOpenAITestMixin): model = "openbmb/MiniCPM-o-2_6" extra_args = [ diff --git a/test/registered/vlm/test_vlm_input_format.py b/test/registered/vlm/test_vlm_input_format.py index 94e387b10..3f059f0f2 100644 --- a/test/registered/vlm/test_vlm_input_format.py +++ b/test/registered/vlm/test_vlm_input_format.py @@ -590,6 +590,7 @@ class TestInternVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTes return dict(processor_output, format="processor_output") +@unittest.skip("temporarily disabled: NaN in next_token_logits") class TestMiniCPMVUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase): model_path = "openbmb/MiniCPM-V-4" chat_template = "minicpmv"