From e95c2e73bd92b16ccf3bf1adf241b34ed37e971b Mon Sep 17 00:00:00 2001 From: Mick Date: Wed, 15 Apr 2026 10:12:29 +0800 Subject: [PATCH] [diffusion] CI: refactor diffusion ci and reduce redundancy (#22810) --- .github/workflows/pr-test-multimodal-gen.yml | 73 +----- .github/workflows/pr-test.yml | 1 + .../sglang/multimodal_gen/test/run_suite.py | 62 +++-- .../test/server/accuracy_testcase_configs.py | 78 ++++++ .../server/ascend/testcase_configs_npu.py | 7 - .../test/server/test_accuracy_2_gpu_a.py | 37 --- .../test/server/test_accuracy_2_gpu_b.py | 37 --- ..._a.py => test_component_accuracy_1_gpu.py} | 28 +- ..._b.py => test_component_accuracy_2_gpu.py} | 28 +- ...{test_server_a.py => test_server_1_gpu.py} | 4 +- ...server_2_gpu_a.py => test_server_2_gpu.py} | 4 +- .../test/server/test_server_2_gpu_b.py | 25 -- .../test/server/test_server_b.py | 31 --- .../test/server/test_server_common.py | 91 +++++-- .../test/server/testcase_configs.py | 242 ++++-------------- .../sglang/multimodal_gen/test/test_utils.py | 2 +- .../utils/diffusion/diffusion_case_parser.py | 19 +- scripts/ci/utils/slash_command_handler.py | 1 + 18 files changed, 306 insertions(+), 464 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/server/accuracy_testcase_configs.py delete mode 100644 python/sglang/multimodal_gen/test/server/test_accuracy_2_gpu_a.py delete mode 100644 python/sglang/multimodal_gen/test/server/test_accuracy_2_gpu_b.py rename python/sglang/multimodal_gen/test/server/{test_accuracy_1_gpu_a.py => test_component_accuracy_1_gpu.py} (59%) rename python/sglang/multimodal_gen/test/server/{test_accuracy_1_gpu_b.py => test_component_accuracy_2_gpu.py} (59%) rename python/sglang/multimodal_gen/test/server/{test_server_a.py => test_server_1_gpu.py} (90%) rename python/sglang/multimodal_gen/test/server/{test_server_2_gpu_a.py => test_server_2_gpu.py} (86%) delete mode 100644 python/sglang/multimodal_gen/test/server/test_server_2_gpu_b.py delete mode 100644 python/sglang/multimodal_gen/test/server/test_server_b.py diff --git a/.github/workflows/pr-test-multimodal-gen.yml b/.github/workflows/pr-test-multimodal-gen.yml index 356b5254d..fe3ddbbbf 100644 --- a/.github/workflows/pr-test-multimodal-gen.yml +++ b/.github/workflows/pr-test-multimodal-gen.yml @@ -221,64 +221,13 @@ jobs: with: artifact-suffix: ${{ matrix.part }} - multimodal-gen-component-accuracy-1-gpu: + multimodal-gen-component-accuracy: if: | - (inputs.target_stage == 'multimodal-gen-component-accuracy-1-gpu') || ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) && - inputs.multimodal_gen == 'true' - ) - runs-on: 1-gpu-h100 - timeout-minutes: 240 - strategy: - fail-fast: false - matrix: - part: [0, 1] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: inputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda12.9 - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion - - - name: Run diffusion component accuracy tests (1-GPU) - timeout-minutes: 240 - env: - RUNAI_STREAMER_MEMORY_LIMIT: 0 - CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd python - python3 sglang/multimodal_gen/test/run_suite.py \ - --suite component-accuracy-1-gpu \ - --partition-id ${{ matrix.part }} \ - --total-partitions 2 \ - $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: always() - with: - artifact-suffix: ${{ matrix.part }} - - multimodal-gen-component-accuracy-2-gpu: - if: | - (inputs.target_stage == 'multimodal-gen-component-accuracy-2-gpu') || + inputs.target_stage == 'multimodal-gen-component-accuracy' || + inputs.target_stage == 'multimodal-gen-component-accuracy-1-gpu' || + inputs.target_stage == 'multimodal-gen-component-accuracy-2-gpu' + ) || ( !inputs.target_stage && ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) && @@ -286,10 +235,6 @@ jobs: ) runs-on: 2-gpu-h100 timeout-minutes: 240 - strategy: - fail-fast: false - matrix: - part: [0, 1] steps: - name: Checkout code uses: actions/checkout@v4 @@ -313,7 +258,7 @@ jobs: run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion - - name: Run diffusion component accuracy tests (2-GPU) + - name: Run diffusion component accuracy tests timeout-minutes: 240 env: RUNAI_STREAMER_MEMORY_LIMIT: 0 @@ -321,15 +266,13 @@ jobs: run: | cd python python3 sglang/multimodal_gen/test/run_suite.py \ - --suite component-accuracy-2-gpu \ - --partition-id ${{ matrix.part }} \ - --total-partitions 2 \ + --suite component-accuracy \ $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps if: always() with: - artifact-suffix: ${{ matrix.part }} + artifact-suffix: component-accuracy multimodal-gen-test-1-b200: if: | diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 8feb364bd..e96f4cdc6 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -891,6 +891,7 @@ jobs: ( inputs.target_stage == 'multimodal-gen-test-1-gpu' || inputs.target_stage == 'multimodal-gen-test-2-gpu' || + inputs.target_stage == 'multimodal-gen-component-accuracy' || inputs.target_stage == 'multimodal-gen-component-accuracy-1-gpu' || inputs.target_stage == 'multimodal-gen-component-accuracy-2-gpu' || inputs.target_stage == 'multimodal-gen-test-1-b200' || diff --git a/python/sglang/multimodal_gen/test/run_suite.py b/python/sglang/multimodal_gen/test/run_suite.py index 851c82a3b..9c1437b20 100644 --- a/python/sglang/multimodal_gen/test/run_suite.py +++ b/python/sglang/multimodal_gen/test/run_suite.py @@ -22,10 +22,8 @@ import tabulate from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.test.server.testcase_configs import ( BASELINE_CONFIG, - ONE_GPU_CASES_A, - ONE_GPU_CASES_B, - TWO_GPU_CASES_A, - TWO_GPU_CASES_B, + ONE_GPU_CASES, + TWO_GPU_CASES, DiffusionTestCase, ) @@ -54,13 +52,15 @@ def _discover_unit_tests() -> list[str]: FILE_SUITES = { "unit": _discover_unit_tests(), + "component-accuracy": [ + "test_component_accuracy_1_gpu.py", + "test_component_accuracy_2_gpu.py", + ], "component-accuracy-1-gpu": [ - "test_accuracy_1_gpu_a.py", - "test_accuracy_1_gpu_b.py", + "test_component_accuracy_1_gpu.py", ], "component-accuracy-2-gpu": [ - "test_accuracy_2_gpu_a.py", - "test_accuracy_2_gpu_b.py", + "test_component_accuracy_2_gpu.py", ], "1-gpu-b200": [ "test_server_c.py", @@ -76,12 +76,10 @@ FILE_SUITES.update(suites_ascend) PARAMETRIZED_CASE_GROUPS = { "1-gpu": [ - ("test_server_a.py", ONE_GPU_CASES_A), - ("test_server_b.py", ONE_GPU_CASES_B), + ("test_server_1_gpu.py", ONE_GPU_CASES), ], "2-gpu": [ - ("test_server_2_gpu_a.py", TWO_GPU_CASES_A), - ("test_server_2_gpu_b.py", TWO_GPU_CASES_B), + ("test_server_2_gpu.py", TWO_GPU_CASES), ], } @@ -116,9 +114,14 @@ SUITES = { STRICT_SUITES = {"unit"} COMPONENT_ACCURACY_SUITES = { + "component-accuracy", "component-accuracy-1-gpu", "component-accuracy-2-gpu", } +COMPONENT_ACCURACY_FILE_NUM_GPUS = { + "test_component_accuracy_1_gpu.py": 1, + "test_component_accuracy_2_gpu.py": 2, +} @dataclass(frozen=True) @@ -540,11 +543,28 @@ def _extract_failure_tail(full_output: str, max_lines: int = 20) -> list[str]: return lines[-max_lines:] +def _summary_has_retryable_failure(summary_lines: list[str]) -> bool: + for line in summary_lines: + lowered = line.lower() + if ( + "[performance]" in line + or "SafetensorError" in line + or "FileNotFoundError" in line + or "TimeoutError" in line + or "out of memory" in lowered + or "oom killer" in lowered + ): + return True + return False + + def _is_retryable_failure(full_output: str) -> bool: + summary_lines = _extract_short_test_summary(full_output) is_perf_assertion = ( "multimodal_gen/test/server/test_server_utils.py" in full_output and "AssertionError" in full_output ) + is_aggregated_retryable_failure = _summary_has_retryable_failure(summary_lines) is_flaky_ci_assertion = ( "SafetensorError" in full_output @@ -556,7 +576,12 @@ def _is_retryable_failure(full_output: str) -> bool: "out of memory" in full_output.lower() or "oom killer" in full_output.lower() ) - return is_perf_assertion or is_flaky_ci_assertion or is_oom_error + return ( + is_perf_assertion + or is_aggregated_retryable_failure + or is_flaky_ci_assertion + or is_oom_error + ) def _print_attempt_tail_summary( @@ -710,17 +735,17 @@ def partition_test_files(files, partition_id, total_partitions): ] -def run_component_accuracy_files( - files, suite: str, filter_expr=None, continue_on_error=False -): +def run_component_accuracy_files(files, filter_expr=None, continue_on_error=False): exit_code = 0 for file_path in files: - if suite == "component-accuracy-2-gpu": + file_name = Path(file_path).name + num_gpus = COMPONENT_ACCURACY_FILE_NUM_GPUS.get(file_name, 1) + if num_gpus > 1: cmd = [ sys.executable, "-m", "torch.distributed.run", - "--nproc_per_node=2", + f"--nproc_per_node={num_gpus}", "-m", "pytest", "-s", @@ -1143,7 +1168,6 @@ def main(): exit_code = run_component_accuracy_files( my_files, - suite=args.suite, filter_expr=args.filter, continue_on_error=args.continue_on_error, ) diff --git a/python/sglang/multimodal_gen/test/server/accuracy_testcase_configs.py b/python/sglang/multimodal_gen/test/server/accuracy_testcase_configs.py new file mode 100644 index 000000000..ed7900219 --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/accuracy_testcase_configs.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from sglang.multimodal_gen.test.server.testcase_configs import ( + ONE_GPU_CASES, + TWO_GPU_CASES, + DiffusionTestCase, +) + + +def _select_accuracy_cases( + cases: list[DiffusionTestCase], enabled_ids: tuple[str, ...] +) -> list[DiffusionTestCase]: + enabled = set(enabled_ids) + return [case for case in cases if case.id in enabled] + + +ACCURACY_ONE_GPU_CASE_IDS = ( + "qwen_image_t2i", + "qwen_image_t2i_cache_dit_enabled", + "flux_image_t2i", + "flux_2_image_t2i", + "flux_2_klein_image_t2i", + "layerwise_offload", + "zimage_image_t2i", + "zimage_image_t2i_fp8", + "zimage_image_t2i_multi_lora", + "qwen_image_edit_ti2i", + "qwen_image_edit_2509_ti2i", + "qwen_image_edit_2511_ti2i", + "qwen_image_layered_i2i", + "flux_2_image_t2i_upscaling_4x", + "mova_360p_1gpu", + "wan2_1_t2v_1.3b", + "wan2_1_t2v_1.3b_text_encoder_cpu_offload", + "wan2_1_t2v_1.3b_teacache_enabled", + "wan2_1_t2v_1.3b_frame_interp_2x", + "wan2_1_t2v_1.3b_upscaling_4x", + "wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x", + "wan2_1_t2v_1_3b_lora_1gpu", + "flux_2_ti2i", + "flux_2_t2i_customized_vae_path", + "fast_hunyuan_video", + "wan2_2_ti2v_5b", + "fastwan2_2_ti2v_5b", + "hunyuan3d_shape_gen", + "turbo_wan2_1_t2v_1.3b", + "flux_2_ti2i_multi_image_cache_dit", +) + +ACCURACY_TWO_GPU_CASE_IDS = ( + "wan2_2_i2v_a14b_2gpu", + "wan2_2_t2v_a14b_2gpu", + "wan2_2_t2v_a14b_teacache_2gpu", + "wan2_2_t2v_a14b_lora_2gpu", + "wan2_1_t2v_14b_2gpu", + "wan2_1_t2v_1.3b_cfg_parallel", + "fsdp-inference", + "mova_360p_tp2", + "mova_360p_ring1_uly2", + "mova_360p_ring2_uly1", + "ltx_2_two_stage_t2v", + "wan2_1_i2v_14b_480P_2gpu", + "wan2_1_i2v_14b_lora_2gpu", + "wan2_1_i2v_14b_720P_2gpu", + "qwen_image_t2i_2_gpus", + "zimage_image_t2i_2_gpus", + "zimage_image_t2i_2_gpus_non_square", + "flux_image_t2i_2_gpus", + "flux_2_image_t2i_2_gpus", + "flux_2_klein_ti2i_2_gpus", +) + +ACCURACY_ONE_GPU_CASES = _select_accuracy_cases( + ONE_GPU_CASES, ACCURACY_ONE_GPU_CASE_IDS +) +ACCURACY_TWO_GPU_CASES = _select_accuracy_cases( + TWO_GPU_CASES, ACCURACY_TWO_GPU_CASE_IDS +) diff --git a/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py b/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py index 4086dcd74..71cdd95ff 100644 --- a/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py +++ b/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py @@ -12,7 +12,6 @@ ONE_NPU_CASES: list[DiffusionTestCase] = [ "flux_image_t2i_npu", DiffusionServerArgs( model_path="/root/.cache/modelscope/hub/models/black-forest-labs/FLUX.1-dev", - modality="image", ), T2I_sampling_params, run_consistency_check=False, @@ -22,8 +21,6 @@ ONE_NPU_CASES: list[DiffusionTestCase] = [ "wan2_1_t2v_1.3b_1_npu", DiffusionServerArgs( model_path="/root/.cache/modelscope/hub/models/Wan-AI/Wan2.1-T2V-1.3B-Diffusers", - modality="video", - custom_validator="video", ), DiffusionSamplingParams( prompt=T2V_PROMPT, @@ -38,7 +35,6 @@ TWO_NPU_CASES: list[DiffusionTestCase] = [ "flux_2_image_t2i_2npu", DiffusionServerArgs( model_path="/root/.cache/modelscope/hub/models/black-forest-labs/FLUX.2-dev", - modality="image", num_gpus=2, tp_size=2, ), @@ -49,7 +45,6 @@ TWO_NPU_CASES: list[DiffusionTestCase] = [ "qwen_image_t2i_2npu", DiffusionServerArgs( model_path="/root/.cache/modelscope/hub/models/Qwen/Qwen-Image", - modality="image", num_gpus=2, # test ring attn ulysses_degree=1, @@ -66,8 +61,6 @@ EIGHT_NPU_CASES: list[DiffusionTestCase] = [ "wan2_2_t2v_14b_w8a8_8npu", DiffusionServerArgs( model_path="/root/.cache/modelscope/hub/models/Eco-Tech/Wan2.2-T2V-A14B-Diffusers-w8a8", - modality="video", - custom_validator="video", num_gpus=8, tp_size=4, ), diff --git a/python/sglang/multimodal_gen/test/server/test_accuracy_2_gpu_a.py b/python/sglang/multimodal_gen/test/server/test_accuracy_2_gpu_a.py deleted file mode 100644 index cd0677a58..000000000 --- a/python/sglang/multimodal_gen/test/server/test_accuracy_2_gpu_a.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest - -from sglang.multimodal_gen.test.server.accuracy_config import ( - ComponentType, - get_skip_reason, - should_skip_component, -) -from sglang.multimodal_gen.test.server.accuracy_utils import ( - run_native_component_accuracy_case, - run_text_encoder_accuracy_case, -) -from sglang.multimodal_gen.test.server.component_accuracy import AccuracyEngine -from sglang.multimodal_gen.test.server.testcase_configs import ACCURACY_TWO_GPU_CASES_A - - -@pytest.mark.parametrize("case", ACCURACY_TWO_GPU_CASES_A, ids=lambda x: x.id) -class TestAccuracy2GPU_A: - """2-GPU Component Accuracy Suite (Set A).""" - - def test_vae_accuracy(self, case): - if should_skip_component(case, ComponentType.VAE): - pytest.skip(get_skip_reason(case, ComponentType.VAE)) - run_native_component_accuracy_case( - AccuracyEngine, case, ComponentType.VAE, "diffusers", 2 - ) - - def test_transformer_accuracy(self, case): - if should_skip_component(case, ComponentType.TRANSFORMER): - pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER)) - run_native_component_accuracy_case( - AccuracyEngine, case, ComponentType.TRANSFORMER, "diffusers", 2 - ) - - def test_encoder_accuracy(self, case): - if should_skip_component(case, ComponentType.TEXT_ENCODER): - pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER)) - run_text_encoder_accuracy_case(AccuracyEngine, case, 2) diff --git a/python/sglang/multimodal_gen/test/server/test_accuracy_2_gpu_b.py b/python/sglang/multimodal_gen/test/server/test_accuracy_2_gpu_b.py deleted file mode 100644 index 5d75cbd93..000000000 --- a/python/sglang/multimodal_gen/test/server/test_accuracy_2_gpu_b.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest - -from sglang.multimodal_gen.test.server.accuracy_config import ( - ComponentType, - get_skip_reason, - should_skip_component, -) -from sglang.multimodal_gen.test.server.accuracy_utils import ( - run_native_component_accuracy_case, - run_text_encoder_accuracy_case, -) -from sglang.multimodal_gen.test.server.component_accuracy import AccuracyEngine -from sglang.multimodal_gen.test.server.testcase_configs import ACCURACY_TWO_GPU_CASES_B - - -@pytest.mark.parametrize("case", ACCURACY_TWO_GPU_CASES_B, ids=lambda x: x.id) -class TestAccuracy2GPU_B: - """2-GPU Component Accuracy Suite (Set B).""" - - def test_vae_accuracy(self, case): - if should_skip_component(case, ComponentType.VAE): - pytest.skip(get_skip_reason(case, ComponentType.VAE)) - run_native_component_accuracy_case( - AccuracyEngine, case, ComponentType.VAE, "diffusers", 2 - ) - - def test_transformer_accuracy(self, case): - if should_skip_component(case, ComponentType.TRANSFORMER): - pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER)) - run_native_component_accuracy_case( - AccuracyEngine, case, ComponentType.TRANSFORMER, "diffusers", 2 - ) - - def test_encoder_accuracy(self, case): - if should_skip_component(case, ComponentType.TEXT_ENCODER): - pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER)) - run_text_encoder_accuracy_case(AccuracyEngine, case, 2) diff --git a/python/sglang/multimodal_gen/test/server/test_accuracy_1_gpu_a.py b/python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py similarity index 59% rename from python/sglang/multimodal_gen/test/server/test_accuracy_1_gpu_a.py rename to python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py index 16aaa4a13..12d606d8a 100644 --- a/python/sglang/multimodal_gen/test/server/test_accuracy_1_gpu_a.py +++ b/python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py @@ -5,33 +5,47 @@ from sglang.multimodal_gen.test.server.accuracy_config import ( get_skip_reason, should_skip_component, ) +from sglang.multimodal_gen.test.server.accuracy_testcase_configs import ( + ACCURACY_ONE_GPU_CASES, +) from sglang.multimodal_gen.test.server.accuracy_utils import ( run_native_component_accuracy_case, run_text_encoder_accuracy_case, ) from sglang.multimodal_gen.test.server.component_accuracy import AccuracyEngine -from sglang.multimodal_gen.test.server.testcase_configs import ACCURACY_ONE_GPU_CASES_A -@pytest.mark.parametrize("case", ACCURACY_ONE_GPU_CASES_A, ids=lambda x: x.id) -class TestAccuracy1GPU_A: - """1-GPU Component Accuracy Suite (Set A).""" +@pytest.mark.parametrize("case", ACCURACY_ONE_GPU_CASES, ids=lambda case: case.id) +class TestComponentAccuracy1GPU: + """1-GPU component accuracy suite.""" def test_vae_accuracy(self, case): if should_skip_component(case, ComponentType.VAE): pytest.skip(get_skip_reason(case, ComponentType.VAE)) run_native_component_accuracy_case( - AccuracyEngine, case, ComponentType.VAE, "diffusers", 1 + AccuracyEngine, + case, + ComponentType.VAE, + "diffusers", + case.server_args.num_gpus, ) def test_transformer_accuracy(self, case): if should_skip_component(case, ComponentType.TRANSFORMER): pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER)) run_native_component_accuracy_case( - AccuracyEngine, case, ComponentType.TRANSFORMER, "diffusers", 1 + AccuracyEngine, + case, + ComponentType.TRANSFORMER, + "diffusers", + case.server_args.num_gpus, ) def test_encoder_accuracy(self, case): if should_skip_component(case, ComponentType.TEXT_ENCODER): pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER)) - run_text_encoder_accuracy_case(AccuracyEngine, case, 1) + run_text_encoder_accuracy_case( + AccuracyEngine, + case, + case.server_args.num_gpus, + ) diff --git a/python/sglang/multimodal_gen/test/server/test_accuracy_1_gpu_b.py b/python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py similarity index 59% rename from python/sglang/multimodal_gen/test/server/test_accuracy_1_gpu_b.py rename to python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py index 810809f33..af85fb65f 100644 --- a/python/sglang/multimodal_gen/test/server/test_accuracy_1_gpu_b.py +++ b/python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py @@ -5,33 +5,47 @@ from sglang.multimodal_gen.test.server.accuracy_config import ( get_skip_reason, should_skip_component, ) +from sglang.multimodal_gen.test.server.accuracy_testcase_configs import ( + ACCURACY_TWO_GPU_CASES, +) from sglang.multimodal_gen.test.server.accuracy_utils import ( run_native_component_accuracy_case, run_text_encoder_accuracy_case, ) from sglang.multimodal_gen.test.server.component_accuracy import AccuracyEngine -from sglang.multimodal_gen.test.server.testcase_configs import ACCURACY_ONE_GPU_CASES_B -@pytest.mark.parametrize("case", ACCURACY_ONE_GPU_CASES_B, ids=lambda x: x.id) -class TestAccuracy1GPU_B: - """1-GPU Component Accuracy Suite (Set B).""" +@pytest.mark.parametrize("case", ACCURACY_TWO_GPU_CASES, ids=lambda case: case.id) +class TestComponentAccuracy2GPU: + """2-GPU component accuracy suite.""" def test_vae_accuracy(self, case): if should_skip_component(case, ComponentType.VAE): pytest.skip(get_skip_reason(case, ComponentType.VAE)) run_native_component_accuracy_case( - AccuracyEngine, case, ComponentType.VAE, "diffusers", 1 + AccuracyEngine, + case, + ComponentType.VAE, + "diffusers", + case.server_args.num_gpus, ) def test_transformer_accuracy(self, case): if should_skip_component(case, ComponentType.TRANSFORMER): pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER)) run_native_component_accuracy_case( - AccuracyEngine, case, ComponentType.TRANSFORMER, "diffusers", 1 + AccuracyEngine, + case, + ComponentType.TRANSFORMER, + "diffusers", + case.server_args.num_gpus, ) def test_encoder_accuracy(self, case): if should_skip_component(case, ComponentType.TEXT_ENCODER): pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER)) - run_text_encoder_accuracy_case(AccuracyEngine, case, 1) + run_text_encoder_accuracy_case( + AccuracyEngine, + case, + case.server_args.num_gpus, + ) diff --git a/python/sglang/multimodal_gen/test/server/test_server_a.py b/python/sglang/multimodal_gen/test/server/test_server_1_gpu.py similarity index 90% rename from python/sglang/multimodal_gen/test/server/test_server_a.py rename to python/sglang/multimodal_gen/test/server/test_server_1_gpu.py index fdf072ec8..54ca6ee5d 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_a.py +++ b/python/sglang/multimodal_gen/test/server/test_server_1_gpu.py @@ -15,7 +15,7 @@ from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 diffusion_server, ) from sglang.multimodal_gen.test.server.testcase_configs import ( - ONE_GPU_CASES_A, + ONE_GPU_CASES, DiffusionTestCase, ) @@ -25,7 +25,7 @@ logger = init_logger(__name__) class TestDiffusionServerOneGpu(DiffusionServerBase): """Performance tests for 1-GPU diffusion cases.""" - @pytest.fixture(params=ONE_GPU_CASES_A, ids=lambda c: c.id) + @pytest.fixture(params=ONE_GPU_CASES, ids=lambda c: c.id) def case(self, request) -> DiffusionTestCase: """Provide a DiffusionTestCase for each 1-GPU test.""" return request.param diff --git a/python/sglang/multimodal_gen/test/server/test_server_2_gpu_a.py b/python/sglang/multimodal_gen/test/server/test_server_2_gpu.py similarity index 86% rename from python/sglang/multimodal_gen/test/server/test_server_2_gpu_a.py rename to python/sglang/multimodal_gen/test/server/test_server_2_gpu.py index 3668f63e6..5b4e5f37d 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_2_gpu_a.py +++ b/python/sglang/multimodal_gen/test/server/test_server_2_gpu.py @@ -11,7 +11,7 @@ from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 diffusion_server, ) from sglang.multimodal_gen.test.server.testcase_configs import ( - TWO_GPU_CASES_A, + TWO_GPU_CASES, DiffusionTestCase, ) @@ -19,7 +19,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import ( class TestDiffusionServerTwoGpu(DiffusionServerBase): """Performance tests for 2-GPU diffusion cases.""" - @pytest.fixture(params=TWO_GPU_CASES_A, ids=lambda c: c.id) + @pytest.fixture(params=TWO_GPU_CASES, ids=lambda c: c.id) def case(self, request) -> DiffusionTestCase: """Provide a DiffusionTestCase for each 2-GPU test.""" return request.param diff --git a/python/sglang/multimodal_gen/test/server/test_server_2_gpu_b.py b/python/sglang/multimodal_gen/test/server/test_server_2_gpu_b.py deleted file mode 100644 index 2c9b5cdc7..000000000 --- a/python/sglang/multimodal_gen/test/server/test_server_2_gpu_b.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -2 GPU tests -""" - -from __future__ import annotations - -import pytest - -from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 - DiffusionServerBase, - diffusion_server, -) -from sglang.multimodal_gen.test.server.testcase_configs import ( - TWO_GPU_CASES_B, - DiffusionTestCase, -) - - -class TestDiffusionServerTwoGpu(DiffusionServerBase): - """Performance tests for 2-GPU diffusion cases.""" - - @pytest.fixture(params=TWO_GPU_CASES_B, ids=lambda c: c.id) - def case(self, request) -> DiffusionTestCase: - """Provide a DiffusionTestCase for each 2-GPU test.""" - return request.param diff --git a/python/sglang/multimodal_gen/test/server/test_server_b.py b/python/sglang/multimodal_gen/test/server/test_server_b.py deleted file mode 100644 index 1a0432db6..000000000 --- a/python/sglang/multimodal_gen/test/server/test_server_b.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Config-driven diffusion performance test with pytest parametrization. - - -If the actual run is significantly better than the baseline, the improved cases with their updated baseline will be printed -""" - -from __future__ import annotations - -import pytest - -from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger -from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 - DiffusionServerBase, - diffusion_server, -) -from sglang.multimodal_gen.test.server.testcase_configs import ( - ONE_GPU_CASES_B, - DiffusionTestCase, -) - -logger = init_logger(__name__) - - -class TestDiffusionServerOneGpu(DiffusionServerBase): - """Performance tests for 1-GPU diffusion cases.""" - - @pytest.fixture(params=ONE_GPU_CASES_B, ids=lambda c: c.id) - def case(self, request) -> DiffusionTestCase: - """Provide a DiffusionTestCase for each 1-GPU test.""" - return request.param diff --git a/python/sglang/multimodal_gen/test/server/test_server_common.py b/python/sglang/multimodal_gen/test/server/test_server_common.py index 956231c5c..2f18b3d32 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_common.py +++ b/python/sglang/multimodal_gen/test/server/test_server_common.py @@ -1090,46 +1090,91 @@ Repository: https://github.com/sglang-bot/sglang-ci-data (path: diffusion-ci/con self._save_gt_output(case, content) return - # Validation 1: Performance - self._validate_and_record(case, perf_record) + failures: list[tuple[str, str]] = [] + + def run_case_check(name: str, fn: Callable[[], None]) -> None: + try: + fn() + except BaseException as exc: + if isinstance(exc, (KeyboardInterrupt, SystemExit)): + raise + failures.append((name, str(exc))) + + run_case_check( + "performance", + lambda: self._validate_and_record(case, perf_record), + ) - # Mesh correctness check (Chamfer Distance) for 3D models if case.server_args.custom_validator == "mesh": from sglang.multimodal_gen.test.server.test_server_utils import ( MESH_OUTPUT_PATHS, validate_mesh_correctness, ) - mesh_path = MESH_OUTPUT_PATHS.pop(case.id, None) - if mesh_path: - validate_mesh_correctness(mesh_path) + def validate_mesh_output() -> None: + mesh_path = MESH_OUTPUT_PATHS.pop(case.id, None) + if mesh_path: + validate_mesh_correctness(mesh_path) + + run_case_check("mesh correctness", validate_mesh_output) - # Test /v1/models endpoint for router compatibility if case.run_models_api_check: - self._test_v1_models_endpoint(diffusion_server, case) + run_case_check( + "/v1/models endpoint", + lambda: self._test_v1_models_endpoint(diffusion_server, case), + ) if case.run_t2v_input_reference_check: - self._test_t2v_rejects_input_reference(diffusion_server, case) + run_case_check( + "t2v input_reference rejection", + lambda: self._test_t2v_rejects_input_reference(diffusion_server, case), + ) if case.run_consistency_check: - self._validate_consistency(case, content) + run_case_check( + "consistency", + lambda: self._validate_consistency(case, content), + ) - # LoRA API functionality test with E2E validation (only for LoRA-enabled cases) if case.run_lora_basic_api_check: - self._test_lora_api_functionality(diffusion_server, case, generate_fn) + run_case_check( + "LoRA basic API", + lambda: self._test_lora_api_functionality( + diffusion_server, case, generate_fn + ), + ) if case.run_lora_dynamic_switch_check: - self._test_lora_dynamic_switch_e2e( - diffusion_server, - case, - generate_fn, - case.server_args.second_lora_path, + run_case_check( + "LoRA dynamic switch", + lambda: self._test_lora_dynamic_switch_e2e( + diffusion_server, + case, + generate_fn, + case.server_args.second_lora_path, + ), ) if case.run_multi_lora_api_check: - self._test_multi_lora_e2e( - diffusion_server, - case, - generate_fn, - case.server_args.lora_path, - case.server_args.second_lora_path, + run_case_check( + "multi-LoRA API", + lambda: self._test_multi_lora_e2e( + diffusion_server, + case, + generate_fn, + case.server_args.lora_path, + case.server_args.second_lora_path, + ), + ) + + if failures: + formatted_failures = [] + for name, message in failures: + if "\n" in message: + formatted_failures.append(f"[{name}]\n{message}") + else: + formatted_failures.append(f"[{name}] {message}") + pytest.fail( + f"Diffusion testcase '{case.id}' failed {len(failures)} check(s):\n\n" + + "\n\n".join(formatted_failures), + pytrace=False, ) diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index b132547e9..d215a74c1 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -3,10 +3,10 @@ Configuration and data structures for diffusion performance tests. Usage: -pytest python/sglang/multimodal_gen/test/server/test_server_a.py -# for a single testcase, look for the name of the testcase in ONE_GPU_CASES_A, -# ONE_GPU_CASES_B, ONE_GPU_CASES_C, TWO_GPU_CASES_A, or TWO_GPU_CASES_B -pytest python/sglang/multimodal_gen/test/server/test_server_a.py -k qwen_image_t2i +pytest python/sglang/multimodal_gen/test/server/test_server_1_gpu.py +# for a single testcase, look for the name of the testcase in ONE_GPU_CASES, +# ONE_GPU_CASES_C, or TWO_GPU_CASES +pytest python/sglang/multimodal_gen/test/server/test_server_1_gpu.py -k qwen_image_t2i To add a new testcase: @@ -22,10 +22,13 @@ from __future__ import annotations import json import os import statistics -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace +from functools import lru_cache from pathlib import Path from typing import Sequence +from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType +from sglang.multimodal_gen.registry import get_model_info from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord from sglang.multimodal_gen.test.test_utils import ( @@ -179,9 +182,9 @@ class DiffusionServerArgs: """Configuration for a single model/scenario test case.""" model_path: str # HF repo or local path - modality: str = "image" # "image" or "video" or "3d" + modality: str | None = None # auto-inferred: "image" or "video" or "3d" - custom_validator: str | None = None # optional custom validator name + custom_validator: str | None = None # auto-derived unless explicitly overridden # resources num_gpus: int = 1 tp_size: int | None = None @@ -208,6 +211,12 @@ class DiffusionServerArgs: extras: list[str] = field(default_factory=lambda: []) def __post_init__(self): + if self.modality is None: + self.modality = _infer_modality_from_model_path(self.model_path) + + if self.custom_validator is not None: + return + if self.modality == "image": self.custom_validator = "image" elif self.modality == "video": @@ -216,6 +225,20 @@ class DiffusionServerArgs: self.custom_validator = "mesh" +@lru_cache(maxsize=None) +def _infer_modality_from_model_path(model_path: str) -> str: + model_info = get_model_info(model_path) + if model_info is None: + raise ValueError(f"Could not resolve model info for {model_path!r}") + + task_type = model_info.pipeline_config_cls.task_type + if task_type == ModelTaskType.I2M: + return "3d" + if task_type.is_image_gen(): + return "image" + return "video" + + @dataclass(frozen=True) class DiffusionSamplingParams: """Configuration for a single model/scenario test case.""" @@ -416,7 +439,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [ "qwen_image_t2i", DiffusionServerArgs( model_path=DEFAULT_QWEN_IMAGE_MODEL_NAME_FOR_TEST, - modality="image", ), T2I_sampling_params, ), @@ -424,16 +446,13 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [ "qwen_image_t2i_cache_dit_enabled", DiffusionServerArgs( model_path=DEFAULT_QWEN_IMAGE_MODEL_NAME_FOR_TEST, - modality="image", enable_cache_dit=True, ), T2I_sampling_params, ), DiffusionTestCase( "flux_image_t2i", - DiffusionServerArgs( - model_path=DEFAULT_FLUX_1_DEV_MODEL_NAME_FOR_TEST, modality="image" - ), + DiffusionServerArgs(model_path=DEFAULT_FLUX_1_DEV_MODEL_NAME_FOR_TEST), T2I_sampling_params, ), # TODO: modeling of flux different from official flux, so weights can't be loaded @@ -441,23 +460,20 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [ # DiffusionTestCase( # "flux_image_t2i_override_transformer_weights_path_fp8", # DiffusionServerArgs( - # model_path="black-forest-labs/FLUX.1-dev", modality="image", + # model_path="black-forest-labs/FLUX.1-dev", # extras=["--transformer-weights-path black-forest-labs/FLUX.1-dev-FP8"] # ), # T2I_sampling_params, # ), DiffusionTestCase( "flux_2_image_t2i", - DiffusionServerArgs( - model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST, modality="image" - ), + DiffusionServerArgs(model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST), T2I_sampling_params, ), DiffusionTestCase( "flux_2_klein_image_t2i", DiffusionServerArgs( model_path=DEFAULT_FLUX_2_KLEIN_4B_MODEL_NAME_FOR_TEST, - modality="image", ), T2I_sampling_params, ), @@ -468,7 +484,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [ "layerwise_offload", DiffusionServerArgs( model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, - modality="image", dit_layerwise_offload=True, dit_offload_prefetch_size=2, ), @@ -476,16 +491,13 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [ ), DiffusionTestCase( "zimage_image_t2i", - DiffusionServerArgs( - model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, modality="image" - ), + DiffusionServerArgs(model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST), T2I_sampling_params, ), DiffusionTestCase( "zimage_image_t2i_fp8", DiffusionServerArgs( model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, - modality="image", extras=["--transformer-path MickJ/Z-Image-Turbo-fp8"], ), T2I_sampling_params, @@ -495,7 +507,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [ "zimage_image_t2i_multi_lora", DiffusionServerArgs( model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, - modality="image", lora_path="reverentelusarca/elusarca-anime-style-lora-z-image-turbo", second_lora_path="tarn59/pixel_art_style_lora_z_image_turbo", ), @@ -507,16 +518,13 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [ # === Text and Image to Image (TI2I) === DiffusionTestCase( "qwen_image_edit_ti2i", - DiffusionServerArgs( - model_path=DEFAULT_QWEN_IMAGE_EDIT_MODEL_NAME_FOR_TEST, modality="image" - ), + DiffusionServerArgs(model_path=DEFAULT_QWEN_IMAGE_EDIT_MODEL_NAME_FOR_TEST), TI2I_sampling_params, ), DiffusionTestCase( "qwen_image_edit_2509_ti2i", DiffusionServerArgs( model_path=DEFAULT_QWEN_IMAGE_EDIT_2509_MODEL_NAME_FOR_TEST, - modality="image", ), MULTI_IMAGE_TI2I_sampling_params, ), @@ -524,7 +532,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [ "qwen_image_edit_2511_ti2i", DiffusionServerArgs( model_path=DEFAULT_QWEN_IMAGE_EDIT_2511_MODEL_NAME_FOR_TEST, - modality="image", ), TI2I_sampling_params, ), @@ -532,7 +539,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [ "qwen_image_layered_i2i", DiffusionServerArgs( model_path=DEFAULT_QWEN_IMAGE_LAYERED_MODEL_NAME_FOR_TEST, - modality="image", ), MULTI_FRAME_I2I_sampling_params, ), @@ -541,7 +547,6 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [ "flux_2_image_t2i_upscaling_4x", DiffusionServerArgs( model_path="black-forest-labs/FLUX.2-dev", - modality="image", ), DiffusionSamplingParams( prompt="Doraemon is eating dorayaki", @@ -562,8 +567,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ "wan2_1_t2v_1.3b", DiffusionServerArgs( model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", ), T2V_sampling_params, ), @@ -571,8 +574,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ "wan2_1_t2v_1.3b_text_encoder_cpu_offload", DiffusionServerArgs( model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", text_encoder_cpu_offload=True, ), T2V_sampling_params, @@ -582,8 +583,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ "wan2_1_t2v_1.3b_teacache_enabled", DiffusionServerArgs( model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", ), DiffusionSamplingParams( prompt=T2V_PROMPT, @@ -596,8 +595,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ "wan2_1_t2v_1.3b_frame_interp_2x", DiffusionServerArgs( model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", - modality="video", - custom_validator="video", ), DiffusionSamplingParams( prompt=T2V_PROMPT, @@ -610,8 +607,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ "wan2_1_t2v_1.3b_upscaling_4x", DiffusionServerArgs( model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", - modality="video", - custom_validator="video", ), DiffusionSamplingParams( prompt=T2V_PROMPT, @@ -624,8 +619,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ "wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x", DiffusionServerArgs( model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", - modality="video", - custom_validator="video", ), DiffusionSamplingParams( prompt=T2V_PROMPT, @@ -644,8 +637,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ "wan2_1_t2v_1_3b_lora_1gpu", DiffusionServerArgs( model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", num_gpus=1, dynamic_lora_path="Cseti/Wan-LoRA-Arcane-Jinx-v1", ), @@ -660,7 +651,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ # "hunyuan_video", # DiffusionServerArgs( # model_path="hunyuanvideo-community/HunyuanVideo", - # modality="video", # ), # DiffusionSamplingParams( # prompt=T2V_PROMPT, @@ -668,16 +658,13 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ # ), DiffusionTestCase( "flux_2_ti2i", - DiffusionServerArgs( - model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST, modality="image" - ), + DiffusionServerArgs(model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST), TI2I_sampling_params, ), DiffusionTestCase( "flux_2_t2i_customized_vae_path", DiffusionServerArgs( model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST, - modality="image", extras=["--vae-path=fal/FLUX.2-Tiny-AutoEncoder"], ), T2I_sampling_params, @@ -687,8 +674,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ "fast_hunyuan_video", DiffusionServerArgs( model_path="FastVideo/FastHunyuan-diffusers", - modality="video", - custom_validator="video", ), T2V_sampling_params, ), @@ -697,8 +682,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ "wan2_2_ti2v_5b", DiffusionServerArgs( model_path=DEFAULT_WAN_2_2_TI2V_5B_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", ), TI2V_sampling_params, ), @@ -706,8 +689,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ "fastwan2_2_ti2v_5b", DiffusionServerArgs( model_path="FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers", - modality="video", - custom_validator="video", ), TI2V_sampling_params, ), @@ -717,7 +698,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ # "helios_base_t2v", # DiffusionServerArgs( # model_path="BestWishYsh/Helios-Base", - # modality="video", # ), # DiffusionSamplingParams( # prompt=T2V_PROMPT, @@ -729,7 +709,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ # "helios_mid_t2v", # DiffusionServerArgs( # model_path="BestWishYsh/Helios-Mid", - # modality="video", # ), # DiffusionSamplingParams( # prompt=T2V_PROMPT, @@ -741,7 +720,6 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [ # "helios_distilled_t2v", # DiffusionServerArgs( # model_path="BestWishYsh/Helios-Distilled", - # modality="video", # ), # DiffusionSamplingParams( # prompt=T2V_PROMPT, @@ -758,7 +736,6 @@ if not current_platform.is_hip(): "hunyuan3d_shape_gen", DiffusionServerArgs( model_path="tencent/Hunyuan3D-2", - modality="3d", enable_warmup=False, ), HUNYUAN3D_SHAPE_sampling_params, @@ -772,8 +749,6 @@ if not current_platform.is_hip(): "turbo_wan2_1_t2v_1.3b", DiffusionServerArgs( model_path="IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers", - modality="video", - custom_validator="video", ), T2V_sampling_params, ) @@ -785,7 +760,6 @@ ONE_GPU_CASES_C = [ "flux_2_nvfp4_t2i", DiffusionServerArgs( model_path="black-forest-labs/FLUX.2-dev-NVFP4", - modality="image", ), T2I_sampling_params, ) @@ -796,8 +770,6 @@ TWO_GPU_CASES_A = [ "wan2_2_i2v_a14b_2gpu", DiffusionServerArgs( model_path=DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", ), TI2V_sampling_params, ), @@ -805,9 +777,6 @@ TWO_GPU_CASES_A = [ "wan2_2_t2v_a14b_2gpu", DiffusionServerArgs( model_path=DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", - num_gpus=2, extras=["--ulysses-degree=2"], ), T2V_sampling_params, @@ -819,9 +788,6 @@ TWO_GPU_CASES_A = [ "wan2_2_t2v_a14b_teacache_2gpu", DiffusionServerArgs( model_path=DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", - num_gpus=2, extras=["--ulysses-degree=2"], ), DiffusionSamplingParams( @@ -835,9 +801,6 @@ TWO_GPU_CASES_A = [ "wan2_2_t2v_a14b_lora_2gpu", DiffusionServerArgs( model_path=DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", - num_gpus=2, lora_path="Cseti/wan2.2-14B-Arcane_Jinx-lora-v1", extras=[ "--lora-weight-name", @@ -853,9 +816,6 @@ TWO_GPU_CASES_A = [ "wan2_1_t2v_14b_2gpu", DiffusionServerArgs( model_path=DEFAULT_WAN_2_1_T2V_14B_MODEL_NAME_FOR_TEST, - modality="video", - num_gpus=2, - custom_validator="video", ), DiffusionSamplingParams( prompt=T2V_PROMPT, @@ -866,9 +826,6 @@ TWO_GPU_CASES_A = [ "wan2_1_t2v_1.3b_cfg_parallel", DiffusionServerArgs( model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", - num_gpus=2, cfg_parallel=True, ), T2V_sampling_params, @@ -877,8 +834,6 @@ TWO_GPU_CASES_A = [ "fsdp-inference", DiffusionServerArgs( model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, - modality="image", - num_gpus=2, extras=["--use-fsdp-inference"], ), T2I_sampling_params, @@ -887,8 +842,6 @@ TWO_GPU_CASES_A = [ "mova_360p_tp2", DiffusionServerArgs( model_path=DEFAULT_MOVA_360P_MODEL_NAME_FOR_TEST, - modality="video", - num_gpus=2, tp_size=2, dit_layerwise_offload=True, ), @@ -899,8 +852,6 @@ TWO_GPU_CASES_A = [ "mova_360p_ring1_uly2", DiffusionServerArgs( model_path=DEFAULT_MOVA_360P_MODEL_NAME_FOR_TEST, - modality="video", - num_gpus=2, ring_degree=1, ulysses_degree=2, dit_layerwise_offload=True, @@ -912,9 +863,9 @@ TWO_GPU_CASES_A = [ "ltx_2_two_stage_t2v", DiffusionServerArgs( model_path="Lightricks/LTX-2", - modality="video", - num_gpus=2, - extras=["--pipeline-class-name LTX2TwoStagePipeline", "--ulysses-degree=2"], + ulysses_degree=2, + dit_layerwise_offload=True, + extras=["--pipeline-class-name LTX2TwoStagePipeline"], ), T2V_sampling_params, ), @@ -922,8 +873,6 @@ TWO_GPU_CASES_A = [ "ltx_2_3_two_stage_ti2v_2gpus", DiffusionServerArgs( model_path="Lightricks/LTX-2.3", - modality="video", - num_gpus=2, extras=["--pipeline-class-name LTX2TwoStagePipeline"], ), TI2V_sampling_params, @@ -935,9 +884,6 @@ TWO_GPU_CASES_B = [ "wan2_1_i2v_14b_480P_2gpu", DiffusionServerArgs( model_path=DEFAULT_WAN_2_1_I2V_14B_480P_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", - num_gpus=2, extras=["--ulysses-degree=2"], ), TI2V_sampling_params, @@ -946,8 +892,6 @@ TWO_GPU_CASES_B = [ "ltx_2.3_two_stage_t2v_2gpus", DiffusionServerArgs( model_path="Lightricks/LTX-2.3", - modality="video", - num_gpus=2, extras=["--pipeline-class-name LTX2TwoStagePipeline"], ), T2V_sampling_params, @@ -957,9 +901,6 @@ TWO_GPU_CASES_B = [ "wan2_1_i2v_14b_lora_2gpu", DiffusionServerArgs( model_path=DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", - num_gpus=2, lora_path="starsfriday/Wan2.1-Divine-Power-LoRA", extras=["--ulysses-degree=2"], ), @@ -970,9 +911,6 @@ TWO_GPU_CASES_B = [ "wan2_1_i2v_14b_720P_2gpu", DiffusionServerArgs( model_path=DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST, - modality="video", - custom_validator="video", - num_gpus=2, extras=["--ulysses-degree=2"], ), TI2V_sampling_params, @@ -981,8 +919,6 @@ TWO_GPU_CASES_B = [ "qwen_image_t2i_2_gpus", DiffusionServerArgs( model_path=DEFAULT_QWEN_IMAGE_MODEL_NAME_FOR_TEST, - modality="image", - num_gpus=2, # test ring attn ulysses_degree=1, ring_degree=2, @@ -993,8 +929,6 @@ TWO_GPU_CASES_B = [ "zimage_image_t2i_2_gpus", DiffusionServerArgs( model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, - modality="image", - num_gpus=2, ulysses_degree=2, ), T2I_sampling_params, @@ -1003,8 +937,6 @@ TWO_GPU_CASES_B = [ "zimage_image_t2i_2_gpus_non_square", DiffusionServerArgs( model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, - modality="image", - num_gpus=2, ulysses_degree=2, ), DiffusionSamplingParams( @@ -1017,8 +949,6 @@ TWO_GPU_CASES_B = [ "flux_image_t2i_2_gpus", DiffusionServerArgs( model_path=DEFAULT_FLUX_1_DEV_MODEL_NAME_FOR_TEST, - modality="image", - num_gpus=2, ), T2I_sampling_params, ), @@ -1026,8 +956,6 @@ TWO_GPU_CASES_B = [ "flux_2_image_t2i_2_gpus", DiffusionServerArgs( model_path=DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST, - modality="image", - num_gpus=2, tp_size=2, ), T2I_sampling_params, @@ -1036,8 +964,6 @@ TWO_GPU_CASES_B = [ "flux_2_klein_ti2i_2_gpus", DiffusionServerArgs( model_path="black-forest-labs/FLUX.2-klein-4B", - modality="image", - num_gpus=2, ), TI2I_sampling_params, ), @@ -1045,8 +971,6 @@ TWO_GPU_CASES_B = [ "ltx_2.3_one_stage_ti2v", DiffusionServerArgs( model_path="Lightricks/LTX-2.3", - modality="video", - num_gpus=2, ), TI2V_sampling_params, ), @@ -1059,7 +983,6 @@ if not current_platform.is_hip(): "flux_2_ti2i_multi_image_cache_dit", DiffusionServerArgs( model_path="black-forest-labs/FLUX.2-dev", - modality="image", enable_cache_dit=True, ), MULTI_IMAGE_TI2I_UPLOAD_sampling_params, @@ -1067,86 +990,19 @@ if not current_platform.is_hip(): ) -def _select_accuracy_cases( - cases: list[DiffusionTestCase], enabled_ids: tuple[str, ...] +def _with_default_num_gpus( + cases: list[DiffusionTestCase], num_gpus: int ) -> list[DiffusionTestCase]: - enabled = set(enabled_ids) - return [case for case in cases if case.id in enabled] + return [ + replace(case, server_args=replace(case.server_args, num_gpus=num_gpus)) + for case in cases + ] -ACCURACY_ONE_GPU_CASES_A_IDS = ( - "qwen_image_t2i", - "qwen_image_t2i_cache_dit_enabled", - "flux_image_t2i", - "flux_2_image_t2i", - "flux_2_klein_image_t2i", - "layerwise_offload", - "zimage_image_t2i", - "zimage_image_t2i_fp8", - "zimage_image_t2i_multi_lora", - "qwen_image_edit_ti2i", - "qwen_image_edit_2509_ti2i", - "qwen_image_edit_2511_ti2i", - "qwen_image_layered_i2i", - "flux_2_image_t2i_upscaling_4x", -) - -ACCURACY_ONE_GPU_CASES_B_IDS = ( - "wan2_1_t2v_1.3b", - "wan2_1_t2v_1.3b_text_encoder_cpu_offload", - "wan2_1_t2v_1.3b_teacache_enabled", - "wan2_1_t2v_1.3b_frame_interp_2x", - "wan2_1_t2v_1.3b_upscaling_4x", - "wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x", - "wan2_1_t2v_1_3b_lora_1gpu", - "flux_2_ti2i", - "flux_2_t2i_customized_vae_path", - "fast_hunyuan_video", - "wan2_2_ti2v_5b", - "fastwan2_2_ti2v_5b", - "hunyuan3d_shape_gen", - "turbo_wan2_1_t2v_1.3b", - "flux_2_nvfp4_t2i", - "flux_2_ti2i_multi_image_cache_dit", -) - -ACCURACY_TWO_GPU_CASES_A_IDS = ( - "wan2_2_i2v_a14b_2gpu", - "wan2_2_t2v_a14b_2gpu", - "wan2_2_t2v_a14b_teacache_2gpu", - "wan2_2_t2v_a14b_lora_2gpu", - "wan2_1_t2v_14b_2gpu", - "wan2_1_t2v_1.3b_cfg_parallel", - "fsdp-inference", - "mova_360p_tp2", - "mova_360p_ring1_uly2", - "ltx_2_two_stage_t2v", -) - -ACCURACY_TWO_GPU_CASES_B_IDS = ( - "wan2_1_i2v_14b_480P_2gpu", - "wan2_1_i2v_14b_lora_2gpu", - "wan2_1_i2v_14b_720P_2gpu", - "qwen_image_t2i_2_gpus", - "zimage_image_t2i_2_gpus", - "zimage_image_t2i_2_gpus_non_square", - "flux_image_t2i_2_gpus", - "flux_2_image_t2i_2_gpus", - "flux_2_klein_ti2i_2_gpus", -) - -ACCURACY_ONE_GPU_CASES_A = _select_accuracy_cases( - ONE_GPU_CASES_A, ACCURACY_ONE_GPU_CASES_A_IDS -) -ACCURACY_ONE_GPU_CASES_B = _select_accuracy_cases( - ONE_GPU_CASES_B, ACCURACY_ONE_GPU_CASES_B_IDS -) -ACCURACY_TWO_GPU_CASES_A = _select_accuracy_cases( - TWO_GPU_CASES_A, ACCURACY_TWO_GPU_CASES_A_IDS -) -ACCURACY_TWO_GPU_CASES_B = _select_accuracy_cases( - TWO_GPU_CASES_B, ACCURACY_TWO_GPU_CASES_B_IDS -) +ONE_GPU_CASES = [*ONE_GPU_CASES_A, *ONE_GPU_CASES_B, *ONE_GPU_CASES_C] +TWO_GPU_CASES_A = _with_default_num_gpus(TWO_GPU_CASES_A, 2) +TWO_GPU_CASES_B = _with_default_num_gpus(TWO_GPU_CASES_B, 2) +TWO_GPU_CASES = [*TWO_GPU_CASES_A, *TWO_GPU_CASES_B] # Load global configuration BASELINE_CONFIG = BaselineConfig.load( diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index cda892d7c..a7ed952ef 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -1129,7 +1129,7 @@ def compare_with_gt( status = "PASSED" if passed else "FAILED" print(f"\n{'=' * 60}") - print(f"[CLIP Consistency] {case_id}: {status}") + print(f"[Consistency Check] {case_id}: {status}") print( " Thresholds: " f"clip>={thresholds.clip_threshold}, " diff --git a/scripts/ci/utils/diffusion/diffusion_case_parser.py b/scripts/ci/utils/diffusion/diffusion_case_parser.py index 0dabeadf3..d5839714e 100755 --- a/scripts/ci/utils/diffusion/diffusion_case_parser.py +++ b/scripts/ci/utils/diffusion/diffusion_case_parser.py @@ -92,16 +92,19 @@ class DiffusionTestCaseVisitor(ast.NodeVisitor): if isinstance(target, ast.Name) and target.id in CASE_LIST_TO_SUITE: list_name = target.id case_ids = self._extract_case_ids_from_list(value) - self.cases[list_name] = case_ids + if case_ids is not None: + self.cases[list_name] = case_ids + + def _extract_case_ids_from_list(self, node: ast.AST) -> Optional[List[str]]: + """Extract case IDs from a literal list of DiffusionTestCase calls.""" + if not isinstance(node, ast.List): + return None - def _extract_case_ids_from_list(self, node: ast.AST) -> List[str]: - """Extract case IDs from a list of DiffusionTestCase calls.""" case_ids = [] - if isinstance(node, ast.List): - for elt in node.elts: - case_id = self._extract_case_id_from_call(elt) - if case_id: - case_ids.append(case_id) + for elt in node.elts: + case_id = self._extract_case_id_from_call(elt) + if case_id: + case_ids.append(case_id) return case_ids def _extract_case_id_from_call(self, node: ast.AST) -> Optional[str]: diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index 74bdd0893..1869ef075 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -299,6 +299,7 @@ def handle_rerun_stage( "stage-c-test-deepep-8-gpu-h200", "multimodal-gen-test-1-gpu", "multimodal-gen-test-2-gpu", + "multimodal-gen-component-accuracy", "multimodal-gen-component-accuracy-1-gpu", "multimodal-gen-component-accuracy-2-gpu", "multimodal-gen-test-1-b200",