diff --git a/test/registered/gb300/test_qwen35_fp8.py b/test/registered/gb300/test_qwen35_fp8.py index f2551aacc..8a2d4b177 100644 --- a/test/registered/gb300/test_qwen35_fp8.py +++ b/test/registered/gb300/test_qwen35_fp8.py @@ -36,6 +36,11 @@ DP_MTP_ARGS = [ "--speculative-num-draft-tokens=2", ] +PERFORMANCE_BATCH_SIZES = { + "TP4+MTP": [1, 4], + "TP4+DP4+DPA+MTP": [16], +} + class TestQwen35Fp8(unittest.TestCase): """Qwen3.5-397B FP8 on GB300 (4x GB300 NVL4, tp=4).""" @@ -58,25 +63,35 @@ class TestQwen35Fp8(unittest.TestCase): ), ] - run_combined_tests( - models=variants, - test_name="Qwen3.5-397B-FP8", - # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly -- - # its `:1` suffix means temperature 0.7, not greedy -- so the baseline - # carries over unchanged. Do not "simplify" these away. - accuracy_params=AccuracyTestParams( - dataset="mmmu_pro_vision", - baseline_accuracy=0.76, - repeat=1, - max_tokens=32768, - temperature=0.7, - seed=0, - sgl_eval_thinking=False, - ), - performance_params=PerformanceTestParams( - result_dir="performance_results_gb300", - ), + failures = [] + # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly -- + # its `:1` suffix means temperature 0.7, not greedy -- so the baseline + # carries over unchanged. Do not "simplify" these away. + accuracy_params = AccuracyTestParams( + dataset="mmmu_pro_vision", + baseline_accuracy=0.76, + repeat=1, + max_tokens=32768, + temperature=0.7, + seed=0, + sgl_eval_thinking=False, ) + for variant in variants: + try: + run_combined_tests( + models=[variant], + test_name=f"Qwen3.5-397B-FP8 ({variant.variant})", + accuracy_params=accuracy_params, + performance_params=PerformanceTestParams( + batch_sizes=PERFORMANCE_BATCH_SIZES[variant.variant], + result_dir="performance_results_gb300", + ), + ) + except AssertionError as e: + failures.append(f"{variant.variant}: {e}") + + if failures: + raise AssertionError("Qwen3.5-397B-FP8 failures:\n" + "\n".join(failures)) if __name__ == "__main__":