[diffusion] CI: refactor diffusion ci and reduce redundancy (#22810)

.github/workflows/pr-test-multimodal-gen.yml

@@ -221,64 +221,13 @@ jobs:
         with:
           artifact-suffix: ${{ matrix.part }}
 
-  multimodal-gen-component-accuracy-1-gpu:
+  multimodal-gen-component-accuracy:
     if: |
-      (inputs.target_stage == 'multimodal-gen-component-accuracy-1-gpu') ||
       (
-        !inputs.target_stage &&
+        inputs.target_stage == 'multimodal-gen-component-accuracy' ||
-        ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
+        inputs.target_stage == 'multimodal-gen-component-accuracy-1-gpu' ||
-        inputs.multimodal_gen == 'true'
+        inputs.target_stage == 'multimodal-gen-component-accuracy-2-gpu'
-      )
+      ) ||
-    runs-on: 1-gpu-h100
-    timeout-minutes: 240
-    strategy:
-      fail-fast: false
-      matrix:
-        part: [0, 1]
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }}
-
-      - uses: ./.github/actions/check-stage-health
-
-      - uses: ./.github/actions/check-maintenance
-
-      - name: Download artifacts
-        if: inputs.sgl_kernel == 'true'
-        uses: actions/download-artifact@v4
-        with:
-          path: sgl-kernel/dist/
-          merge-multiple: true
-          pattern: wheel-python3.10-cuda12.9
-
-      - name: Install dependencies
-        timeout-minutes: 20
-        run: |
-          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
-
-      - name: Run diffusion component accuracy tests (1-GPU)
-        timeout-minutes: 240
-        env:
-          RUNAI_STREAMER_MEMORY_LIMIT: 0
-          CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
-        run: |
-          cd python
-          python3 sglang/multimodal_gen/test/run_suite.py \
-            --suite component-accuracy-1-gpu \
-            --partition-id ${{ matrix.part }} \
-            --total-partitions 2 \
-            $CONTINUE_ON_ERROR_FLAG
-
-      - uses: ./.github/actions/upload-cuda-coredumps
-        if: always()
-        with:
-          artifact-suffix: ${{ matrix.part }}
-
-  multimodal-gen-component-accuracy-2-gpu:
-    if: |
-      (inputs.target_stage == 'multimodal-gen-component-accuracy-2-gpu') ||
       (
         !inputs.target_stage &&
         ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
@@ -286,10 +235,6 @@ jobs:
       )
     runs-on: 2-gpu-h100
     timeout-minutes: 240
-    strategy:
-      fail-fast: false
-      matrix:
-        part: [0, 1]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -313,7 +258,7 @@ jobs:
         run: |
           CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
 
-      - name: Run diffusion component accuracy tests (2-GPU)
+      - name: Run diffusion component accuracy tests
         timeout-minutes: 240
         env:
           RUNAI_STREAMER_MEMORY_LIMIT: 0
@@ -321,15 +266,13 @@ jobs:
         run: |
           cd python
           python3 sglang/multimodal_gen/test/run_suite.py \
-            --suite component-accuracy-2-gpu \
+            --suite component-accuracy \
-            --partition-id ${{ matrix.part }} \
-            --total-partitions 2 \
             $CONTINUE_ON_ERROR_FLAG
 
       - uses: ./.github/actions/upload-cuda-coredumps
         if: always()
         with:
-          artifact-suffix: ${{ matrix.part }}
+          artifact-suffix: component-accuracy
 
   multimodal-gen-test-1-b200:
     if: |

.github/workflows/pr-test.yml

@@ -891,6 +891,7 @@ jobs:
       (
         inputs.target_stage == 'multimodal-gen-test-1-gpu' ||
         inputs.target_stage == 'multimodal-gen-test-2-gpu' ||
+        inputs.target_stage == 'multimodal-gen-component-accuracy' ||
         inputs.target_stage == 'multimodal-gen-component-accuracy-1-gpu' ||
         inputs.target_stage == 'multimodal-gen-component-accuracy-2-gpu' ||
         inputs.target_stage == 'multimodal-gen-test-1-b200' ||

python/sglang/multimodal_gen/test/run_suite.py

@@ -22,10 +22,8 @@ import tabulate
 from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
 from sglang.multimodal_gen.test.server.testcase_configs import (
     BASELINE_CONFIG,
-    ONE_GPU_CASES_A,
+    ONE_GPU_CASES,
-    ONE_GPU_CASES_B,
+    TWO_GPU_CASES,
-    TWO_GPU_CASES_A,
-    TWO_GPU_CASES_B,
     DiffusionTestCase,
 )
 
@@ -54,13 +52,15 @@ def _discover_unit_tests() -> list[str]:
 
 FILE_SUITES = {
     "unit": _discover_unit_tests(),
+    "component-accuracy": [
+        "test_component_accuracy_1_gpu.py",
+        "test_component_accuracy_2_gpu.py",
+    ],
     "component-accuracy-1-gpu": [
-        "test_accuracy_1_gpu_a.py",
+        "test_component_accuracy_1_gpu.py",
-        "test_accuracy_1_gpu_b.py",
     ],
     "component-accuracy-2-gpu": [
-        "test_accuracy_2_gpu_a.py",
+        "test_component_accuracy_2_gpu.py",
-        "test_accuracy_2_gpu_b.py",
     ],
     "1-gpu-b200": [
         "test_server_c.py",
@@ -76,12 +76,10 @@ FILE_SUITES.update(suites_ascend)
 
 PARAMETRIZED_CASE_GROUPS = {
     "1-gpu": [
-        ("test_server_a.py", ONE_GPU_CASES_A),
+        ("test_server_1_gpu.py", ONE_GPU_CASES),
-        ("test_server_b.py", ONE_GPU_CASES_B),
     ],
     "2-gpu": [
-        ("test_server_2_gpu_a.py", TWO_GPU_CASES_A),
+        ("test_server_2_gpu.py", TWO_GPU_CASES),
-        ("test_server_2_gpu_b.py", TWO_GPU_CASES_B),
     ],
 }
 
@@ -116,9 +114,14 @@ SUITES = {
 
 STRICT_SUITES = {"unit"}
 COMPONENT_ACCURACY_SUITES = {
+    "component-accuracy",
     "component-accuracy-1-gpu",
     "component-accuracy-2-gpu",
 }
+COMPONENT_ACCURACY_FILE_NUM_GPUS = {
+    "test_component_accuracy_1_gpu.py": 1,
+    "test_component_accuracy_2_gpu.py": 2,
+}
 
 
 @dataclass(frozen=True)
@@ -540,11 +543,28 @@ def _extract_failure_tail(full_output: str, max_lines: int = 20) -> list[str]:
     return lines[-max_lines:]
 
 
+def _summary_has_retryable_failure(summary_lines: list[str]) -> bool:
+    for line in summary_lines:
+        lowered = line.lower()
+        if (
+            "[performance]" in line
+            or "SafetensorError" in line
+            or "FileNotFoundError" in line
+            or "TimeoutError" in line
+            or "out of memory" in lowered
+            or "oom killer" in lowered
+        ):
+            return True
+    return False
+
+
 def _is_retryable_failure(full_output: str) -> bool:
+    summary_lines = _extract_short_test_summary(full_output)
     is_perf_assertion = (
         "multimodal_gen/test/server/test_server_utils.py" in full_output
         and "AssertionError" in full_output
     )
+    is_aggregated_retryable_failure = _summary_has_retryable_failure(summary_lines)
 
     is_flaky_ci_assertion = (
         "SafetensorError" in full_output
@@ -556,7 +576,12 @@ def _is_retryable_failure(full_output: str) -> bool:
         "out of memory" in full_output.lower() or "oom killer" in full_output.lower()
     )
 
-    return is_perf_assertion or is_flaky_ci_assertion or is_oom_error
+    return (
+        is_perf_assertion
+        or is_aggregated_retryable_failure
+        or is_flaky_ci_assertion
+        or is_oom_error
+    )
 
 
 def _print_attempt_tail_summary(
@@ -710,17 +735,17 @@ def partition_test_files(files, partition_id, total_partitions):
     ]
 
 
-def run_component_accuracy_files(
+def run_component_accuracy_files(files, filter_expr=None, continue_on_error=False):
-    files, suite: str, filter_expr=None, continue_on_error=False
-):
     exit_code = 0
     for file_path in files:
-        if suite == "component-accuracy-2-gpu":
+        file_name = Path(file_path).name
+        num_gpus = COMPONENT_ACCURACY_FILE_NUM_GPUS.get(file_name, 1)
+        if num_gpus > 1:
             cmd = [
                 sys.executable,
                 "-m",
                 "torch.distributed.run",
-                "--nproc_per_node=2",
+                f"--nproc_per_node={num_gpus}",
                 "-m",
                 "pytest",
                 "-s",
@@ -1143,7 +1168,6 @@ def main():
 
         exit_code = run_component_accuracy_files(
             my_files,
-            suite=args.suite,
             filter_expr=args.filter,
             continue_on_error=args.continue_on_error,
         )

python/sglang/multimodal_gen/test/server/accuracy_testcase_configs.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from sglang.multimodal_gen.test.server.testcase_configs import (
+    ONE_GPU_CASES,
+    TWO_GPU_CASES,
+    DiffusionTestCase,
+)
+
+
+def _select_accuracy_cases(
+    cases: list[DiffusionTestCase], enabled_ids: tuple[str, ...]
+) -> list[DiffusionTestCase]:
+    enabled = set(enabled_ids)
+    return [case for case in cases if case.id in enabled]
+
+
+ACCURACY_ONE_GPU_CASE_IDS = (
+    "qwen_image_t2i",
+    "qwen_image_t2i_cache_dit_enabled",
+    "flux_image_t2i",
+    "flux_2_image_t2i",
+    "flux_2_klein_image_t2i",
+    "layerwise_offload",
+    "zimage_image_t2i",
+    "zimage_image_t2i_fp8",
+    "zimage_image_t2i_multi_lora",
+    "qwen_image_edit_ti2i",
+    "qwen_image_edit_2509_ti2i",
+    "qwen_image_edit_2511_ti2i",
+    "qwen_image_layered_i2i",
+    "flux_2_image_t2i_upscaling_4x",
+    "mova_360p_1gpu",
+    "wan2_1_t2v_1.3b",
+    "wan2_1_t2v_1.3b_text_encoder_cpu_offload",
+    "wan2_1_t2v_1.3b_teacache_enabled",
+    "wan2_1_t2v_1.3b_frame_interp_2x",
+    "wan2_1_t2v_1.3b_upscaling_4x",
+    "wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x",
+    "wan2_1_t2v_1_3b_lora_1gpu",
+    "flux_2_ti2i",
+    "flux_2_t2i_customized_vae_path",
+    "fast_hunyuan_video",
+    "wan2_2_ti2v_5b",
+    "fastwan2_2_ti2v_5b",
+    "hunyuan3d_shape_gen",
+    "turbo_wan2_1_t2v_1.3b",
+    "flux_2_ti2i_multi_image_cache_dit",
+)
+
+ACCURACY_TWO_GPU_CASE_IDS = (
+    "wan2_2_i2v_a14b_2gpu",
+    "wan2_2_t2v_a14b_2gpu",
+    "wan2_2_t2v_a14b_teacache_2gpu",
+    "wan2_2_t2v_a14b_lora_2gpu",
+    "wan2_1_t2v_14b_2gpu",
+    "wan2_1_t2v_1.3b_cfg_parallel",
+    "fsdp-inference",
+    "mova_360p_tp2",
+    "mova_360p_ring1_uly2",
+    "mova_360p_ring2_uly1",
+    "ltx_2_two_stage_t2v",
+    "wan2_1_i2v_14b_480P_2gpu",
+    "wan2_1_i2v_14b_lora_2gpu",
+    "wan2_1_i2v_14b_720P_2gpu",
+    "qwen_image_t2i_2_gpus",
+    "zimage_image_t2i_2_gpus",
+    "zimage_image_t2i_2_gpus_non_square",
+    "flux_image_t2i_2_gpus",
+    "flux_2_image_t2i_2_gpus",
+    "flux_2_klein_ti2i_2_gpus",
+)
+
+ACCURACY_ONE_GPU_CASES = _select_accuracy_cases(
+    ONE_GPU_CASES, ACCURACY_ONE_GPU_CASE_IDS
+)
+ACCURACY_TWO_GPU_CASES = _select_accuracy_cases(
+    TWO_GPU_CASES, ACCURACY_TWO_GPU_CASE_IDS
+)

python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py

@@ -12,7 +12,6 @@ ONE_NPU_CASES: list[DiffusionTestCase] = [
         "flux_image_t2i_npu",
         DiffusionServerArgs(
             model_path="/root/.cache/modelscope/hub/models/black-forest-labs/FLUX.1-dev",
-            modality="image",
         ),
         T2I_sampling_params,
         run_consistency_check=False,
@@ -22,8 +21,6 @@ ONE_NPU_CASES: list[DiffusionTestCase] = [
         "wan2_1_t2v_1.3b_1_npu",
         DiffusionServerArgs(
             model_path="/root/.cache/modelscope/hub/models/Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
-            modality="video",
-            custom_validator="video",
         ),
         DiffusionSamplingParams(
             prompt=T2V_PROMPT,
@@ -38,7 +35,6 @@ TWO_NPU_CASES: list[DiffusionTestCase] = [
         "flux_2_image_t2i_2npu",
         DiffusionServerArgs(
             model_path="/root/.cache/modelscope/hub/models/black-forest-labs/FLUX.2-dev",
-            modality="image",
             num_gpus=2,
             tp_size=2,
         ),
@@ -49,7 +45,6 @@ TWO_NPU_CASES: list[DiffusionTestCase] = [
         "qwen_image_t2i_2npu",
         DiffusionServerArgs(
             model_path="/root/.cache/modelscope/hub/models/Qwen/Qwen-Image",
-            modality="image",
             num_gpus=2,
             # test ring attn
             ulysses_degree=1,
@@ -66,8 +61,6 @@ EIGHT_NPU_CASES: list[DiffusionTestCase] = [
         "wan2_2_t2v_14b_w8a8_8npu",
         DiffusionServerArgs(
             model_path="/root/.cache/modelscope/hub/models/Eco-Tech/Wan2.2-T2V-A14B-Diffusers-w8a8",
-            modality="video",
-            custom_validator="video",
             num_gpus=8,
             tp_size=4,
         ),

python/sglang/multimodal_gen/test/server/test_accuracy_2_gpu_a.py

@@ -1,37 +0,0 @@
-import pytest
-
-from sglang.multimodal_gen.test.server.accuracy_config import (
-    ComponentType,
-    get_skip_reason,
-    should_skip_component,
-)
-from sglang.multimodal_gen.test.server.accuracy_utils import (
-    run_native_component_accuracy_case,
-    run_text_encoder_accuracy_case,
-)
-from sglang.multimodal_gen.test.server.component_accuracy import AccuracyEngine
-from sglang.multimodal_gen.test.server.testcase_configs import ACCURACY_TWO_GPU_CASES_A
-
-
-@pytest.mark.parametrize("case", ACCURACY_TWO_GPU_CASES_A, ids=lambda x: x.id)
-class TestAccuracy2GPU_A:
-    """2-GPU Component Accuracy Suite (Set A)."""
-
-    def test_vae_accuracy(self, case):
-        if should_skip_component(case, ComponentType.VAE):
-            pytest.skip(get_skip_reason(case, ComponentType.VAE))
-        run_native_component_accuracy_case(
-            AccuracyEngine, case, ComponentType.VAE, "diffusers", 2
-        )
-
-    def test_transformer_accuracy(self, case):
-        if should_skip_component(case, ComponentType.TRANSFORMER):
-            pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER))
-        run_native_component_accuracy_case(
-            AccuracyEngine, case, ComponentType.TRANSFORMER, "diffusers", 2
-        )
-
-    def test_encoder_accuracy(self, case):
-        if should_skip_component(case, ComponentType.TEXT_ENCODER):
-            pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER))
-        run_text_encoder_accuracy_case(AccuracyEngine, case, 2)

python/sglang/multimodal_gen/test/server/test_accuracy_2_gpu_b.py

@@ -1,37 +0,0 @@
-import pytest
-
-from sglang.multimodal_gen.test.server.accuracy_config import (
-    ComponentType,
-    get_skip_reason,
-    should_skip_component,
-)
-from sglang.multimodal_gen.test.server.accuracy_utils import (
-    run_native_component_accuracy_case,
-    run_text_encoder_accuracy_case,
-)
-from sglang.multimodal_gen.test.server.component_accuracy import AccuracyEngine
-from sglang.multimodal_gen.test.server.testcase_configs import ACCURACY_TWO_GPU_CASES_B
-
-
-@pytest.mark.parametrize("case", ACCURACY_TWO_GPU_CASES_B, ids=lambda x: x.id)
-class TestAccuracy2GPU_B:
-    """2-GPU Component Accuracy Suite (Set B)."""
-
-    def test_vae_accuracy(self, case):
-        if should_skip_component(case, ComponentType.VAE):
-            pytest.skip(get_skip_reason(case, ComponentType.VAE))
-        run_native_component_accuracy_case(
-            AccuracyEngine, case, ComponentType.VAE, "diffusers", 2
-        )
-
-    def test_transformer_accuracy(self, case):
-        if should_skip_component(case, ComponentType.TRANSFORMER):
-            pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER))
-        run_native_component_accuracy_case(
-            AccuracyEngine, case, ComponentType.TRANSFORMER, "diffusers", 2
-        )
-
-    def test_encoder_accuracy(self, case):
-        if should_skip_component(case, ComponentType.TEXT_ENCODER):
-            pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER))
-        run_text_encoder_accuracy_case(AccuracyEngine, case, 2)

python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py

@@ -5,33 +5,47 @@ from sglang.multimodal_gen.test.server.accuracy_config import (
     get_skip_reason,
     should_skip_component,
 )
+from sglang.multimodal_gen.test.server.accuracy_testcase_configs import (
+    ACCURACY_ONE_GPU_CASES,
+)
 from sglang.multimodal_gen.test.server.accuracy_utils import (
     run_native_component_accuracy_case,
     run_text_encoder_accuracy_case,
 )
 from sglang.multimodal_gen.test.server.component_accuracy import AccuracyEngine
-from sglang.multimodal_gen.test.server.testcase_configs import ACCURACY_ONE_GPU_CASES_A
 
 
-@pytest.mark.parametrize("case", ACCURACY_ONE_GPU_CASES_A, ids=lambda x: x.id)
+@pytest.mark.parametrize("case", ACCURACY_ONE_GPU_CASES, ids=lambda case: case.id)
-class TestAccuracy1GPU_A:
+class TestComponentAccuracy1GPU:
-    """1-GPU Component Accuracy Suite (Set A)."""
+    """1-GPU component accuracy suite."""
 
     def test_vae_accuracy(self, case):
         if should_skip_component(case, ComponentType.VAE):
             pytest.skip(get_skip_reason(case, ComponentType.VAE))
         run_native_component_accuracy_case(
-            AccuracyEngine, case, ComponentType.VAE, "diffusers", 1
+            AccuracyEngine,
+            case,
+            ComponentType.VAE,
+            "diffusers",
+            case.server_args.num_gpus,
         )
 
     def test_transformer_accuracy(self, case):
         if should_skip_component(case, ComponentType.TRANSFORMER):
             pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER))
         run_native_component_accuracy_case(
-            AccuracyEngine, case, ComponentType.TRANSFORMER, "diffusers", 1
+            AccuracyEngine,
+            case,
+            ComponentType.TRANSFORMER,
+            "diffusers",
+            case.server_args.num_gpus,
         )
 
     def test_encoder_accuracy(self, case):
         if should_skip_component(case, ComponentType.TEXT_ENCODER):
             pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER))
-        run_text_encoder_accuracy_case(AccuracyEngine, case, 1)
+        run_text_encoder_accuracy_case(
+            AccuracyEngine,
+            case,
+            case.server_args.num_gpus,
+        )

python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py

@@ -5,33 +5,47 @@ from sglang.multimodal_gen.test.server.accuracy_config import (
     get_skip_reason,
     should_skip_component,
 )
+from sglang.multimodal_gen.test.server.accuracy_testcase_configs import (
+    ACCURACY_TWO_GPU_CASES,
+)
 from sglang.multimodal_gen.test.server.accuracy_utils import (
     run_native_component_accuracy_case,
     run_text_encoder_accuracy_case,
 )
 from sglang.multimodal_gen.test.server.component_accuracy import AccuracyEngine
-from sglang.multimodal_gen.test.server.testcase_configs import ACCURACY_ONE_GPU_CASES_B
 
 
-@pytest.mark.parametrize("case", ACCURACY_ONE_GPU_CASES_B, ids=lambda x: x.id)
+@pytest.mark.parametrize("case", ACCURACY_TWO_GPU_CASES, ids=lambda case: case.id)
-class TestAccuracy1GPU_B:
+class TestComponentAccuracy2GPU:
-    """1-GPU Component Accuracy Suite (Set B)."""
+    """2-GPU component accuracy suite."""
 
     def test_vae_accuracy(self, case):
         if should_skip_component(case, ComponentType.VAE):
             pytest.skip(get_skip_reason(case, ComponentType.VAE))
         run_native_component_accuracy_case(
-            AccuracyEngine, case, ComponentType.VAE, "diffusers", 1
+            AccuracyEngine,
+            case,
+            ComponentType.VAE,
+            "diffusers",
+            case.server_args.num_gpus,
         )
 
     def test_transformer_accuracy(self, case):
         if should_skip_component(case, ComponentType.TRANSFORMER):
             pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER))
         run_native_component_accuracy_case(
-            AccuracyEngine, case, ComponentType.TRANSFORMER, "diffusers", 1
+            AccuracyEngine,
+            case,
+            ComponentType.TRANSFORMER,
+            "diffusers",
+            case.server_args.num_gpus,
         )
 
     def test_encoder_accuracy(self, case):
         if should_skip_component(case, ComponentType.TEXT_ENCODER):
             pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER))
-        run_text_encoder_accuracy_case(AccuracyEngine, case, 1)
+        run_text_encoder_accuracy_case(
+            AccuracyEngine,
+            case,
+            case.server_args.num_gpus,
+        )

python/sglang/multimodal_gen/test/server/test_server_1_gpu.py

@@ -15,7 +15,7 @@ from sglang.multimodal_gen.test.server.test_server_common import (  # noqa: F401
     diffusion_server,
 )
 from sglang.multimodal_gen.test.server.testcase_configs import (
-    ONE_GPU_CASES_A,
+    ONE_GPU_CASES,
     DiffusionTestCase,
 )
 
@@ -25,7 +25,7 @@ logger = init_logger(__name__)
 class TestDiffusionServerOneGpu(DiffusionServerBase):
     """Performance tests for 1-GPU diffusion cases."""
 
-    @pytest.fixture(params=ONE_GPU_CASES_A, ids=lambda c: c.id)
+    @pytest.fixture(params=ONE_GPU_CASES, ids=lambda c: c.id)
     def case(self, request) -> DiffusionTestCase:
         """Provide a DiffusionTestCase for each 1-GPU test."""
         return request.param

python/sglang/multimodal_gen/test/server/test_server_2_gpu.py

@@ -11,7 +11,7 @@ from sglang.multimodal_gen.test.server.test_server_common import (  # noqa: F401
     diffusion_server,
 )
 from sglang.multimodal_gen.test.server.testcase_configs import (
-    TWO_GPU_CASES_A,
+    TWO_GPU_CASES,
     DiffusionTestCase,
 )
 
@@ -19,7 +19,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
 class TestDiffusionServerTwoGpu(DiffusionServerBase):
     """Performance tests for 2-GPU diffusion cases."""
 
-    @pytest.fixture(params=TWO_GPU_CASES_A, ids=lambda c: c.id)
+    @pytest.fixture(params=TWO_GPU_CASES, ids=lambda c: c.id)
     def case(self, request) -> DiffusionTestCase:
         """Provide a DiffusionTestCase for each 2-GPU test."""
         return request.param

python/sglang/multimodal_gen/test/server/test_server_2_gpu_b.py

@@ -1,25 +0,0 @@
-"""
-2 GPU tests
-"""
-
-from __future__ import annotations
-
-import pytest
-
-from sglang.multimodal_gen.test.server.test_server_common import (  # noqa: F401
-    DiffusionServerBase,
-    diffusion_server,
-)
-from sglang.multimodal_gen.test.server.testcase_configs import (
-    TWO_GPU_CASES_B,
-    DiffusionTestCase,
-)
-
-
-class TestDiffusionServerTwoGpu(DiffusionServerBase):
-    """Performance tests for 2-GPU diffusion cases."""
-
-    @pytest.fixture(params=TWO_GPU_CASES_B, ids=lambda c: c.id)
-    def case(self, request) -> DiffusionTestCase:
-        """Provide a DiffusionTestCase for each 2-GPU test."""
-        return request.param

python/sglang/multimodal_gen/test/server/test_server_b.py

@@ -1,31 +0,0 @@
-"""
-Config-driven diffusion performance test with pytest parametrization.
-
-
-If the actual run is significantly better than the baseline, the improved cases with their updated baseline will be printed
-"""
-
-from __future__ import annotations
-
-import pytest
-
-from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
-from sglang.multimodal_gen.test.server.test_server_common import (  # noqa: F401
-    DiffusionServerBase,
-    diffusion_server,
-)
-from sglang.multimodal_gen.test.server.testcase_configs import (
-    ONE_GPU_CASES_B,
-    DiffusionTestCase,
-)
-
-logger = init_logger(__name__)
-
-
-class TestDiffusionServerOneGpu(DiffusionServerBase):
-    """Performance tests for 1-GPU diffusion cases."""
-
-    @pytest.fixture(params=ONE_GPU_CASES_B, ids=lambda c: c.id)
-    def case(self, request) -> DiffusionTestCase:
-        """Provide a DiffusionTestCase for each 1-GPU test."""
-        return request.param

python/sglang/multimodal_gen/test/server/test_server_common.py

@@ -1090,46 +1090,91 @@ Repository: https://github.com/sglang-bot/sglang-ci-data (path: diffusion-ci/con
             self._save_gt_output(case, content)
             return
 
-        # Validation 1: Performance
+        failures: list[tuple[str, str]] = []
-        self._validate_and_record(case, perf_record)
+
+        def run_case_check(name: str, fn: Callable[[], None]) -> None:
+            try:
+                fn()
+            except BaseException as exc:
+                if isinstance(exc, (KeyboardInterrupt, SystemExit)):
+                    raise
+                failures.append((name, str(exc)))
+
+        run_case_check(
+            "performance",
+            lambda: self._validate_and_record(case, perf_record),
+        )
 
-        # Mesh correctness check (Chamfer Distance) for 3D models
         if case.server_args.custom_validator == "mesh":
             from sglang.multimodal_gen.test.server.test_server_utils import (
                 MESH_OUTPUT_PATHS,
                 validate_mesh_correctness,
             )
 
+            def validate_mesh_output() -> None:
                 mesh_path = MESH_OUTPUT_PATHS.pop(case.id, None)
                 if mesh_path:
                     validate_mesh_correctness(mesh_path)
 
-        # Test /v1/models endpoint for router compatibility
+            run_case_check("mesh correctness", validate_mesh_output)
+
         if case.run_models_api_check:
-            self._test_v1_models_endpoint(diffusion_server, case)
+            run_case_check(
+                "/v1/models endpoint",
+                lambda: self._test_v1_models_endpoint(diffusion_server, case),
+            )
         if case.run_t2v_input_reference_check:
-            self._test_t2v_rejects_input_reference(diffusion_server, case)
+            run_case_check(
+                "t2v input_reference rejection",
+                lambda: self._test_t2v_rejects_input_reference(diffusion_server, case),
+            )
 
         if case.run_consistency_check:
-            self._validate_consistency(case, content)
+            run_case_check(
+                "consistency",
+                lambda: self._validate_consistency(case, content),
+            )
 
-        # LoRA API functionality test with E2E validation (only for LoRA-enabled cases)
         if case.run_lora_basic_api_check:
-            self._test_lora_api_functionality(diffusion_server, case, generate_fn)
+            run_case_check(
+                "LoRA basic API",
+                lambda: self._test_lora_api_functionality(
+                    diffusion_server, case, generate_fn
+                ),
+            )
 
         if case.run_lora_dynamic_switch_check:
-            self._test_lora_dynamic_switch_e2e(
+            run_case_check(
+                "LoRA dynamic switch",
+                lambda: self._test_lora_dynamic_switch_e2e(
                     diffusion_server,
                     case,
                     generate_fn,
                     case.server_args.second_lora_path,
+                ),
             )
 
         if case.run_multi_lora_api_check:
-            self._test_multi_lora_e2e(
+            run_case_check(
+                "multi-LoRA API",
+                lambda: self._test_multi_lora_e2e(
                     diffusion_server,
                     case,
                     generate_fn,
                     case.server_args.lora_path,
                     case.server_args.second_lora_path,
+                ),
+            )
+
+        if failures:
+            formatted_failures = []
+            for name, message in failures:
+                if "\n" in message:
+                    formatted_failures.append(f"[{name}]\n{message}")
+                else:
+                    formatted_failures.append(f"[{name}] {message}")
+            pytest.fail(
+                f"Diffusion testcase '{case.id}' failed {len(failures)} check(s):\n\n"
+                + "\n\n".join(formatted_failures),
+                pytrace=False,
             )

python/sglang/multimodal_gen/test/server/testcase_configs.py

@@ -3,10 +3,10 @@ Configuration and data structures for diffusion performance tests.
 
 Usage:
 
-pytest python/sglang/multimodal_gen/test/server/test_server_a.py
+pytest python/sglang/multimodal_gen/test/server/test_server_1_gpu.py
-# for a single testcase, look for the name of the testcase in ONE_GPU_CASES_A,
+# for a single testcase, look for the name of the testcase in ONE_GPU_CASES,
-# ONE_GPU_CASES_B, ONE_GPU_CASES_C, TWO_GPU_CASES_A, or TWO_GPU_CASES_B
+# ONE_GPU_CASES_C, or TWO_GPU_CASES
-pytest python/sglang/multimodal_gen/test/server/test_server_a.py -k qwen_image_t2i
+pytest python/sglang/multimodal_gen/test/server/test_server_1_gpu.py -k qwen_image_t2i
 
 
 To add a new testcase:
@@ -22,10 +22,13 @@ from __future__ import annotations
 import json
 import os
 import statistics
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
+from functools import lru_cache
 from pathlib import Path
 from typing import Sequence
 
+from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType
+from sglang.multimodal_gen.registry import get_model_info
 from sglang.multimodal_gen.runtime.platforms import current_platform
 from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord
 from sglang.multimodal_gen.test.test_utils import (
@@ -179,9 +182,9 @@ class DiffusionServerArgs:
     """Configuration for a single model/scenario test case."""
 
     model_path: str  # HF repo or local path
-    modality: str = "image"  # "image" or "video" or "3d"
+    modality: str | None = None  # auto-inferred: "image" or "video" or "3d"
 
-    custom_validator: str | None = None  # optional custom validator name
+    custom_validator: str | None = None  # auto-derived unless explicitly overridden
     # resources
     num_gpus: int = 1
     tp_size: int | None = None
@@ -208,6 +211,12 @@ class DiffusionServerArgs:
     extras: list[str] = field(default_factory=lambda: [])
 
     def __post_init__(self):
+        if self.modality is None:
+            self.modality = _infer_modality_from_model_path(self.model_path)
+
+        if self.custom_validator is not None:
+            return
+
         if self.modality == "image":
             self.custom_validator = "image"
         elif self.modality == "video":
@@ -216,6 +225,20 @@ class DiffusionServerArgs:
             self.custom_validator = "mesh"
 
 
+@lru_cache(maxsize=None)
+def _infer_modality_from_model_path(model_path: str) -> str:
+    model_info = get_model_info(model_path)
+    if model_info is None:
+        raise ValueError(f"Could not resolve model info for {model_path!r}")
+
+    task_type = model_info.pipeline_config_cls.task_type
+    if task_type == ModelTaskType.I2M:
+        return "3d"
+    if task_type.is_image_gen():
+        return "image"
+    return "video"
+
+
 @dataclass(frozen=True)
 class DiffusionSamplingParams:
     """Configuration for a single model/scenario test case."""
@@ -416,7 +439,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
         "qwen_image_t2i",
         DiffusionServerArgs(
             model_path=DEFAULT_QWEN_IMAGE_MODEL_NAME_FOR_TEST,
-            modality="image",
         ),
         T2I_sampling_params,
     ),
@@ -424,16 +446,13 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
         "qwen_image_t2i_cache_dit_enabled",
         DiffusionServerArgs(
             model_path=DEFAULT_QWEN_IMAGE_MODEL_NAME_FOR_TEST,
-            modality="image",
             enable_cache_dit=True,
         ),
         T2I_sampling_params,
     ),
     DiffusionTestCase(
         "flux_image_t2i",
-        DiffusionServerArgs(
+        DiffusionServerArgs(model_path=DEFAULT_FLUX_1_DEV_MODEL_NAME_FOR_TEST),
-            model_path=DEFAULT_FLUX_1_DEV_MODEL_NAME_FOR_TEST, modality="image"
-        ),
         T2I_sampling_params,
     ),
     # TODO: modeling of flux different from official flux, so weights can't be loaded
@@ -441,23 +460,20 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
     # DiffusionTestCase(
     #     "flux_image_t2i_override_transformer_weights_path_fp8",
     #     DiffusionServerArgs(
-    #         model_path="black-forest-labs/FLUX.1-dev", modality="image",
+    #         model_path="black-forest-labs/FLUX.1-dev",
     #         extras=["--transformer-weights-path black-forest-labs/FLUX.1-dev-FP8"]
     #     ),
     #     T2I_sampling_params,
     # ),
     DiffusionTestCase(
         "flux_2_image_t2i",
-        DiffusionServerArgs(
+        DiffusionServerArgs(model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST),
-            model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST, modality="image"
-        ),
         T2I_sampling_params,
     ),
     DiffusionTestCase(
         "flux_2_klein_image_t2i",
         DiffusionServerArgs(
             model_path=DEFAULT_FLUX_2_KLEIN_4B_MODEL_NAME_FOR_TEST,
-            modality="image",
         ),
         T2I_sampling_params,
     ),
@@ -468,7 +484,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
         "layerwise_offload",
         DiffusionServerArgs(
             model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
-            modality="image",
             dit_layerwise_offload=True,
             dit_offload_prefetch_size=2,
         ),
@@ -476,16 +491,13 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
     ),
     DiffusionTestCase(
         "zimage_image_t2i",
-        DiffusionServerArgs(
+        DiffusionServerArgs(model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST),
-            model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, modality="image"
-        ),
         T2I_sampling_params,
     ),
     DiffusionTestCase(
         "zimage_image_t2i_fp8",
         DiffusionServerArgs(
             model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
-            modality="image",
             extras=["--transformer-path MickJ/Z-Image-Turbo-fp8"],
         ),
         T2I_sampling_params,
@@ -495,7 +507,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
         "zimage_image_t2i_multi_lora",
         DiffusionServerArgs(
             model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
-            modality="image",
             lora_path="reverentelusarca/elusarca-anime-style-lora-z-image-turbo",
             second_lora_path="tarn59/pixel_art_style_lora_z_image_turbo",
         ),
@@ -507,16 +518,13 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
     # === Text and Image to Image (TI2I) ===
     DiffusionTestCase(
         "qwen_image_edit_ti2i",
-        DiffusionServerArgs(
+        DiffusionServerArgs(model_path=DEFAULT_QWEN_IMAGE_EDIT_MODEL_NAME_FOR_TEST),
-            model_path=DEFAULT_QWEN_IMAGE_EDIT_MODEL_NAME_FOR_TEST, modality="image"
-        ),
         TI2I_sampling_params,
     ),
     DiffusionTestCase(
         "qwen_image_edit_2509_ti2i",
         DiffusionServerArgs(
             model_path=DEFAULT_QWEN_IMAGE_EDIT_2509_MODEL_NAME_FOR_TEST,
-            modality="image",
         ),
         MULTI_IMAGE_TI2I_sampling_params,
     ),
@@ -524,7 +532,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
         "qwen_image_edit_2511_ti2i",
         DiffusionServerArgs(
             model_path=DEFAULT_QWEN_IMAGE_EDIT_2511_MODEL_NAME_FOR_TEST,
-            modality="image",
         ),
         TI2I_sampling_params,
     ),
@@ -532,7 +539,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
         "qwen_image_layered_i2i",
         DiffusionServerArgs(
             model_path=DEFAULT_QWEN_IMAGE_LAYERED_MODEL_NAME_FOR_TEST,
-            modality="image",
         ),
         MULTI_FRAME_I2I_sampling_params,
     ),
@@ -541,7 +547,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
         "flux_2_image_t2i_upscaling_4x",
         DiffusionServerArgs(
             model_path="black-forest-labs/FLUX.2-dev",
-            modality="image",
         ),
         DiffusionSamplingParams(
             prompt="Doraemon is eating dorayaki",
@@ -562,8 +567,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
         "wan2_1_t2v_1.3b",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
         ),
         T2V_sampling_params,
     ),
@@ -571,8 +574,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
         "wan2_1_t2v_1.3b_text_encoder_cpu_offload",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
             text_encoder_cpu_offload=True,
         ),
         T2V_sampling_params,
@@ -582,8 +583,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
         "wan2_1_t2v_1.3b_teacache_enabled",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
         ),
         DiffusionSamplingParams(
             prompt=T2V_PROMPT,
@@ -596,8 +595,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
         "wan2_1_t2v_1.3b_frame_interp_2x",
         DiffusionServerArgs(
             model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
-            modality="video",
-            custom_validator="video",
         ),
         DiffusionSamplingParams(
             prompt=T2V_PROMPT,
@@ -610,8 +607,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
         "wan2_1_t2v_1.3b_upscaling_4x",
         DiffusionServerArgs(
             model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
-            modality="video",
-            custom_validator="video",
         ),
         DiffusionSamplingParams(
             prompt=T2V_PROMPT,
@@ -624,8 +619,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
         "wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x",
         DiffusionServerArgs(
             model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
-            modality="video",
-            custom_validator="video",
         ),
         DiffusionSamplingParams(
             prompt=T2V_PROMPT,
@@ -644,8 +637,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
         "wan2_1_t2v_1_3b_lora_1gpu",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
             num_gpus=1,
             dynamic_lora_path="Cseti/Wan-LoRA-Arcane-Jinx-v1",
         ),
@@ -660,7 +651,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
     #     "hunyuan_video",
     #     DiffusionServerArgs(
     #         model_path="hunyuanvideo-community/HunyuanVideo",
-    #         modality="video",
     #     ),
     #     DiffusionSamplingParams(
     #         prompt=T2V_PROMPT,
@@ -668,16 +658,13 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
     # ),
     DiffusionTestCase(
         "flux_2_ti2i",
-        DiffusionServerArgs(
+        DiffusionServerArgs(model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST),
-            model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST, modality="image"
-        ),
         TI2I_sampling_params,
     ),
     DiffusionTestCase(
         "flux_2_t2i_customized_vae_path",
         DiffusionServerArgs(
             model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST,
-            modality="image",
             extras=["--vae-path=fal/FLUX.2-Tiny-AutoEncoder"],
         ),
         T2I_sampling_params,
@@ -687,8 +674,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
         "fast_hunyuan_video",
         DiffusionServerArgs(
             model_path="FastVideo/FastHunyuan-diffusers",
-            modality="video",
-            custom_validator="video",
         ),
         T2V_sampling_params,
     ),
@@ -697,8 +682,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
         "wan2_2_ti2v_5b",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_2_TI2V_5B_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
         ),
         TI2V_sampling_params,
     ),
@@ -706,8 +689,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
         "fastwan2_2_ti2v_5b",
         DiffusionServerArgs(
             model_path="FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
-            modality="video",
-            custom_validator="video",
         ),
         TI2V_sampling_params,
     ),
@@ -717,7 +698,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
     #     "helios_base_t2v",
     #     DiffusionServerArgs(
     #         model_path="BestWishYsh/Helios-Base",
-    #         modality="video",
     #     ),
     #     DiffusionSamplingParams(
     #         prompt=T2V_PROMPT,
@@ -729,7 +709,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
     #     "helios_mid_t2v",
     #     DiffusionServerArgs(
     #         model_path="BestWishYsh/Helios-Mid",
-    #         modality="video",
     #     ),
     #     DiffusionSamplingParams(
     #         prompt=T2V_PROMPT,
@@ -741,7 +720,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
     #     "helios_distilled_t2v",
     #     DiffusionServerArgs(
     #         model_path="BestWishYsh/Helios-Distilled",
-    #         modality="video",
     #     ),
     #     DiffusionSamplingParams(
     #         prompt=T2V_PROMPT,
@@ -758,7 +736,6 @@ if not current_platform.is_hip():
             "hunyuan3d_shape_gen",
             DiffusionServerArgs(
                 model_path="tencent/Hunyuan3D-2",
-                modality="3d",
                 enable_warmup=False,
             ),
             HUNYUAN3D_SHAPE_sampling_params,
@@ -772,8 +749,6 @@ if not current_platform.is_hip():
             "turbo_wan2_1_t2v_1.3b",
             DiffusionServerArgs(
                 model_path="IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers",
-                modality="video",
-                custom_validator="video",
             ),
             T2V_sampling_params,
         )
@@ -785,7 +760,6 @@ ONE_GPU_CASES_C = [
         "flux_2_nvfp4_t2i",
         DiffusionServerArgs(
             model_path="black-forest-labs/FLUX.2-dev-NVFP4",
-            modality="image",
         ),
         T2I_sampling_params,
     )
@@ -796,8 +770,6 @@ TWO_GPU_CASES_A = [
         "wan2_2_i2v_a14b_2gpu",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
         ),
         TI2V_sampling_params,
     ),
@@ -805,9 +777,6 @@ TWO_GPU_CASES_A = [
         "wan2_2_t2v_a14b_2gpu",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
-            num_gpus=2,
             extras=["--ulysses-degree=2"],
         ),
         T2V_sampling_params,
@@ -819,9 +788,6 @@ TWO_GPU_CASES_A = [
         "wan2_2_t2v_a14b_teacache_2gpu",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
-            num_gpus=2,
             extras=["--ulysses-degree=2"],
         ),
         DiffusionSamplingParams(
@@ -835,9 +801,6 @@ TWO_GPU_CASES_A = [
         "wan2_2_t2v_a14b_lora_2gpu",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
-            num_gpus=2,
             lora_path="Cseti/wan2.2-14B-Arcane_Jinx-lora-v1",
             extras=[
                 "--lora-weight-name",
@@ -853,9 +816,6 @@ TWO_GPU_CASES_A = [
         "wan2_1_t2v_14b_2gpu",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_1_T2V_14B_MODEL_NAME_FOR_TEST,
-            modality="video",
-            num_gpus=2,
-            custom_validator="video",
         ),
         DiffusionSamplingParams(
             prompt=T2V_PROMPT,
@@ -866,9 +826,6 @@ TWO_GPU_CASES_A = [
         "wan2_1_t2v_1.3b_cfg_parallel",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
-            num_gpus=2,
             cfg_parallel=True,
         ),
         T2V_sampling_params,
@@ -877,8 +834,6 @@ TWO_GPU_CASES_A = [
         "fsdp-inference",
         DiffusionServerArgs(
             model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
-            modality="image",
-            num_gpus=2,
             extras=["--use-fsdp-inference"],
         ),
         T2I_sampling_params,
@@ -887,8 +842,6 @@ TWO_GPU_CASES_A = [
         "mova_360p_tp2",
         DiffusionServerArgs(
             model_path=DEFAULT_MOVA_360P_MODEL_NAME_FOR_TEST,
-            modality="video",
-            num_gpus=2,
             tp_size=2,
             dit_layerwise_offload=True,
         ),
@@ -899,8 +852,6 @@ TWO_GPU_CASES_A = [
         "mova_360p_ring1_uly2",
         DiffusionServerArgs(
             model_path=DEFAULT_MOVA_360P_MODEL_NAME_FOR_TEST,
-            modality="video",
-            num_gpus=2,
             ring_degree=1,
             ulysses_degree=2,
             dit_layerwise_offload=True,
@@ -912,9 +863,9 @@ TWO_GPU_CASES_A = [
         "ltx_2_two_stage_t2v",
         DiffusionServerArgs(
             model_path="Lightricks/LTX-2",
-            modality="video",
+            ulysses_degree=2,
-            num_gpus=2,
+            dit_layerwise_offload=True,
-            extras=["--pipeline-class-name LTX2TwoStagePipeline", "--ulysses-degree=2"],
+            extras=["--pipeline-class-name LTX2TwoStagePipeline"],
         ),
         T2V_sampling_params,
     ),
@@ -922,8 +873,6 @@ TWO_GPU_CASES_A = [
         "ltx_2_3_two_stage_ti2v_2gpus",
         DiffusionServerArgs(
             model_path="Lightricks/LTX-2.3",
-            modality="video",
-            num_gpus=2,
             extras=["--pipeline-class-name LTX2TwoStagePipeline"],
         ),
         TI2V_sampling_params,
@@ -935,9 +884,6 @@ TWO_GPU_CASES_B = [
         "wan2_1_i2v_14b_480P_2gpu",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_1_I2V_14B_480P_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
-            num_gpus=2,
             extras=["--ulysses-degree=2"],
         ),
         TI2V_sampling_params,
@@ -946,8 +892,6 @@ TWO_GPU_CASES_B = [
         "ltx_2.3_two_stage_t2v_2gpus",
         DiffusionServerArgs(
             model_path="Lightricks/LTX-2.3",
-            modality="video",
-            num_gpus=2,
             extras=["--pipeline-class-name LTX2TwoStagePipeline"],
         ),
         T2V_sampling_params,
@@ -957,9 +901,6 @@ TWO_GPU_CASES_B = [
         "wan2_1_i2v_14b_lora_2gpu",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
-            num_gpus=2,
             lora_path="starsfriday/Wan2.1-Divine-Power-LoRA",
             extras=["--ulysses-degree=2"],
         ),
@@ -970,9 +911,6 @@ TWO_GPU_CASES_B = [
         "wan2_1_i2v_14b_720P_2gpu",
         DiffusionServerArgs(
             model_path=DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST,
-            modality="video",
-            custom_validator="video",
-            num_gpus=2,
             extras=["--ulysses-degree=2"],
         ),
         TI2V_sampling_params,
@@ -981,8 +919,6 @@ TWO_GPU_CASES_B = [
         "qwen_image_t2i_2_gpus",
         DiffusionServerArgs(
             model_path=DEFAULT_QWEN_IMAGE_MODEL_NAME_FOR_TEST,
-            modality="image",
-            num_gpus=2,
             # test ring attn
             ulysses_degree=1,
             ring_degree=2,
@@ -993,8 +929,6 @@ TWO_GPU_CASES_B = [
         "zimage_image_t2i_2_gpus",
         DiffusionServerArgs(
             model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
-            modality="image",
-            num_gpus=2,
             ulysses_degree=2,
         ),
         T2I_sampling_params,
@@ -1003,8 +937,6 @@ TWO_GPU_CASES_B = [
         "zimage_image_t2i_2_gpus_non_square",
         DiffusionServerArgs(
             model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
-            modality="image",
-            num_gpus=2,
             ulysses_degree=2,
         ),
         DiffusionSamplingParams(
@@ -1017,8 +949,6 @@ TWO_GPU_CASES_B = [
         "flux_image_t2i_2_gpus",
         DiffusionServerArgs(
             model_path=DEFAULT_FLUX_1_DEV_MODEL_NAME_FOR_TEST,
-            modality="image",
-            num_gpus=2,
         ),
         T2I_sampling_params,
     ),
@@ -1026,8 +956,6 @@ TWO_GPU_CASES_B = [
         "flux_2_image_t2i_2_gpus",
         DiffusionServerArgs(
             model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST,
-            modality="image",
-            num_gpus=2,
             tp_size=2,
         ),
         T2I_sampling_params,
@@ -1036,8 +964,6 @@ TWO_GPU_CASES_B = [
         "flux_2_klein_ti2i_2_gpus",
         DiffusionServerArgs(
             model_path="black-forest-labs/FLUX.2-klein-4B",
-            modality="image",
-            num_gpus=2,
         ),
         TI2I_sampling_params,
     ),
@@ -1045,8 +971,6 @@ TWO_GPU_CASES_B = [
         "ltx_2.3_one_stage_ti2v",
         DiffusionServerArgs(
             model_path="Lightricks/LTX-2.3",
-            modality="video",
-            num_gpus=2,
         ),
         TI2V_sampling_params,
     ),
@@ -1059,7 +983,6 @@ if not current_platform.is_hip():
             "flux_2_ti2i_multi_image_cache_dit",
             DiffusionServerArgs(
                 model_path="black-forest-labs/FLUX.2-dev",
-                modality="image",
                 enable_cache_dit=True,
             ),
             MULTI_IMAGE_TI2I_UPLOAD_sampling_params,
@@ -1067,86 +990,19 @@ if not current_platform.is_hip():
     )
 
 
-def _select_accuracy_cases(
+def _with_default_num_gpus(
-    cases: list[DiffusionTestCase], enabled_ids: tuple[str, ...]
+    cases: list[DiffusionTestCase], num_gpus: int
 ) -> list[DiffusionTestCase]:
-    enabled = set(enabled_ids)
+    return [
-    return [case for case in cases if case.id in enabled]
+        replace(case, server_args=replace(case.server_args, num_gpus=num_gpus))
+        for case in cases
+    ]
 
 
-ACCURACY_ONE_GPU_CASES_A_IDS = (
+ONE_GPU_CASES = [*ONE_GPU_CASES_A, *ONE_GPU_CASES_B, *ONE_GPU_CASES_C]
-    "qwen_image_t2i",
+TWO_GPU_CASES_A = _with_default_num_gpus(TWO_GPU_CASES_A, 2)
-    "qwen_image_t2i_cache_dit_enabled",
+TWO_GPU_CASES_B = _with_default_num_gpus(TWO_GPU_CASES_B, 2)
-    "flux_image_t2i",
+TWO_GPU_CASES = [*TWO_GPU_CASES_A, *TWO_GPU_CASES_B]
-    "flux_2_image_t2i",
-    "flux_2_klein_image_t2i",
-    "layerwise_offload",
-    "zimage_image_t2i",
-    "zimage_image_t2i_fp8",
-    "zimage_image_t2i_multi_lora",
-    "qwen_image_edit_ti2i",
-    "qwen_image_edit_2509_ti2i",
-    "qwen_image_edit_2511_ti2i",
-    "qwen_image_layered_i2i",
-    "flux_2_image_t2i_upscaling_4x",
-)
-
-ACCURACY_ONE_GPU_CASES_B_IDS = (
-    "wan2_1_t2v_1.3b",
-    "wan2_1_t2v_1.3b_text_encoder_cpu_offload",
-    "wan2_1_t2v_1.3b_teacache_enabled",
-    "wan2_1_t2v_1.3b_frame_interp_2x",
-    "wan2_1_t2v_1.3b_upscaling_4x",
-    "wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x",
-    "wan2_1_t2v_1_3b_lora_1gpu",
-    "flux_2_ti2i",
-    "flux_2_t2i_customized_vae_path",
-    "fast_hunyuan_video",
-    "wan2_2_ti2v_5b",
-    "fastwan2_2_ti2v_5b",
-    "hunyuan3d_shape_gen",
-    "turbo_wan2_1_t2v_1.3b",
-    "flux_2_nvfp4_t2i",
-    "flux_2_ti2i_multi_image_cache_dit",
-)
-
-ACCURACY_TWO_GPU_CASES_A_IDS = (
-    "wan2_2_i2v_a14b_2gpu",
-    "wan2_2_t2v_a14b_2gpu",
-    "wan2_2_t2v_a14b_teacache_2gpu",
-    "wan2_2_t2v_a14b_lora_2gpu",
-    "wan2_1_t2v_14b_2gpu",
-    "wan2_1_t2v_1.3b_cfg_parallel",
-    "fsdp-inference",
-    "mova_360p_tp2",
-    "mova_360p_ring1_uly2",
-    "ltx_2_two_stage_t2v",
-)
-
-ACCURACY_TWO_GPU_CASES_B_IDS = (
-    "wan2_1_i2v_14b_480P_2gpu",
-    "wan2_1_i2v_14b_lora_2gpu",
-    "wan2_1_i2v_14b_720P_2gpu",
-    "qwen_image_t2i_2_gpus",
-    "zimage_image_t2i_2_gpus",
-    "zimage_image_t2i_2_gpus_non_square",
-    "flux_image_t2i_2_gpus",
-    "flux_2_image_t2i_2_gpus",
-    "flux_2_klein_ti2i_2_gpus",
-)
-
-ACCURACY_ONE_GPU_CASES_A = _select_accuracy_cases(
-    ONE_GPU_CASES_A, ACCURACY_ONE_GPU_CASES_A_IDS
-)
-ACCURACY_ONE_GPU_CASES_B = _select_accuracy_cases(
-    ONE_GPU_CASES_B, ACCURACY_ONE_GPU_CASES_B_IDS
-)
-ACCURACY_TWO_GPU_CASES_A = _select_accuracy_cases(
-    TWO_GPU_CASES_A, ACCURACY_TWO_GPU_CASES_A_IDS
-)
-ACCURACY_TWO_GPU_CASES_B = _select_accuracy_cases(
-    TWO_GPU_CASES_B, ACCURACY_TWO_GPU_CASES_B_IDS
-)
 
 # Load global configuration
 BASELINE_CONFIG = BaselineConfig.load(

python/sglang/multimodal_gen/test/test_utils.py

@@ -1129,7 +1129,7 @@ def compare_with_gt(
 
     status = "PASSED" if passed else "FAILED"
     print(f"\n{'=' * 60}")
-    print(f"[CLIP Consistency] {case_id}: {status}")
+    print(f"[Consistency Check] {case_id}: {status}")
     print(
         "  Thresholds: "
         f"clip>={thresholds.clip_threshold}, "

scripts/ci/utils/diffusion/diffusion_case_parser.py

@@ -92,12 +92,15 @@ class DiffusionTestCaseVisitor(ast.NodeVisitor):
             if isinstance(target, ast.Name) and target.id in CASE_LIST_TO_SUITE:
                 list_name = target.id
                 case_ids = self._extract_case_ids_from_list(value)
+                if case_ids is not None:
                     self.cases[list_name] = case_ids
 
-    def _extract_case_ids_from_list(self, node: ast.AST) -> List[str]:
+    def _extract_case_ids_from_list(self, node: ast.AST) -> Optional[List[str]]:
-        """Extract case IDs from a list of DiffusionTestCase calls."""
+        """Extract case IDs from a literal list of DiffusionTestCase calls."""
+        if not isinstance(node, ast.List):
+            return None
+
         case_ids = []
-        if isinstance(node, ast.List):
         for elt in node.elts:
             case_id = self._extract_case_id_from_call(elt)
             if case_id:

scripts/ci/utils/slash_command_handler.py

@@ -299,6 +299,7 @@ def handle_rerun_stage(
         "stage-c-test-deepep-8-gpu-h200",
         "multimodal-gen-test-1-gpu",
         "multimodal-gen-test-2-gpu",
+        "multimodal-gen-component-accuracy",
         "multimodal-gen-component-accuracy-1-gpu",
         "multimodal-gen-component-accuracy-2-gpu",
         "multimodal-gen-test-1-b200",