[diffusion] CI: switch CI data references to sgl-project/ci-data (#24299)

2026-05-03 23:05:12 +08:00
parent c0f5950636
commit 5925572c95
12 changed files with 130 additions and 162 deletions
@@ -40,7 +40,6 @@ DEFAULT_THRESHOLDS = {
 CASE_THRESHOLDS: Dict[str, Dict[ComponentType, float]] = {
    # Add overrides here when a specific model/component needs a different threshold.
    "flux_2_image_t2i": {ComponentType.TRANSFORMER: 0.99},
-    "flux_2_image_t2i_layerwise_offload": {ComponentType.TRANSFORMER: 0.99},
    "flux_2_image_t2i_2_gpus": {ComponentType.TRANSFORMER: 0.99},
    "flux_2_ti2i": {ComponentType.TRANSFORMER: 0.99},
    "flux_2_t2i_customized_vae_path": {ComponentType.TRANSFORMER: 0.99},
@@ -62,11 +61,6 @@ SKIP_COMPONENTS: Dict[str, Dict[ComponentType, ComponentSkip]] = {
            "Text encoder diverges from HF baseline despite 100% matched weights (CosSim ~0.47)"
        )
    },
-    "sana_image_t2i": {
-        ComponentType.VAE: ComponentSkip(
-            "HF AutoencoderDC checkpoint leaves required to_qkv_multiscale weights missing, so VAE transfer would compare against partially initialized reference weights"
-        )
-    },
    "qwen_image_t2i_cache_dit_enabled": {
        ComponentType.VAE: ComponentSkip(
            "Representative VAE accuracy is already covered by qwen_image_t2i for the same source component and topology"
@@ -104,6 +98,9 @@ SKIP_COMPONENTS: Dict[str, Dict[ComponentType, ComponentSkip]] = {
        ComponentType.VAE: ComponentSkip(
            "Representative VAE accuracy is already covered by zimage_image_t2i for the same source component and topology"
        ),
+        ComponentType.TRANSFORMER: ComponentSkip(
+            "FP8 transformer override cannot be materialized by the Diffusers reference loader"
+        ),
        ComponentType.TEXT_ENCODER: ComponentSkip(
            "Representative text encoder accuracy is already covered by zimage_image_t2i for the same source component and topology"
        ),
@@ -1,5 +1,13 @@
 from __future__ import annotations

+from sglang.multimodal_gen.test.server.accuracy_config import (
+    ComponentType,
+    should_skip_component,
+)
+from sglang.multimodal_gen.test.server.accuracy_utils import (
+    extract_component_path_overrides,
+)
+from sglang.multimodal_gen.test.server.component_accuracy import COMPONENT_SPECS
 from sglang.multimodal_gen.test.server.gpu_cases import (
    ONE_GPU_CASES,
    TWO_GPU_CASES,
@@ -7,70 +15,62 @@ from sglang.multimodal_gen.test.server.gpu_cases import (
 from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase


-def _select_accuracy_cases(
-    cases: list[DiffusionTestCase], enabled_ids: tuple[str, ...]
-) -> list[DiffusionTestCase]:
-    enabled = set(enabled_ids)
-    return [case for case in cases if case.id in enabled]
+def _component_accuracy_key(case: DiffusionTestCase, component: ComponentType) -> tuple:
+    server_args = case.server_args
+    component_paths = extract_component_path_overrides(server_args.extras)
+    override_path = None
+    for key in (component.value, *COMPONENT_SPECS[component].model_index_keys):
+        if key in component_paths:
+            override_path = component_paths[key]
+            break
+
+    return (
+        component.value,
+        server_args.model_path,
+        override_path,
+        server_args.num_gpus,
+        server_args.tp_size,
+        server_args.ulysses_degree,
+        server_args.ring_degree,
+        server_args.cfg_parallel,
+    )


-ACCURACY_ONE_GPU_CASE_IDS = (
-    "qwen_image_t2i",
-    "qwen_image_t2i_cache_dit_enabled",
-    "flux_image_t2i",
-    "flux_2_image_t2i",
-    "flux_2_klein_image_t2i",
-    "layerwise_offload",
-    "zimage_image_t2i",
-    "zimage_image_t2i_fp8",
-    "zimage_image_t2i_multi_lora",
-    "qwen_image_edit_ti2i",
-    "qwen_image_edit_2509_ti2i",
-    "qwen_image_edit_2511_ti2i",
-    "qwen_image_layered_i2i",
-    "flux_2_image_t2i_upscaling_4x",
-    "mova_360p_1gpu",
-    "wan2_1_t2v_1.3b",
-    "wan2_1_t2v_1.3b_teacache_enabled",
-    "wan2_1_t2v_1.3b_frame_interp_2x",
-    "wan2_1_t2v_1.3b_upscaling_4x",
-    "wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x",
-    "wan2_1_t2v_1_3b_lora_1gpu",
-    "flux_2_ti2i",
-    "flux_2_t2i_customized_vae_path",
-    "fast_hunyuan_video",
-    "wan2_2_ti2v_5b",
-    "fastwan2_2_ti2v_5b",
-    "hunyuan3d_shape_gen",
-    "turbo_wan2_1_t2v_1.3b",
-    "flux_2_ti2i_multi_image_cache_dit",
-)
+_COMPONENT_DUPLICATE_REASONS: dict[tuple[str, ComponentType], str] = {}

-ACCURACY_TWO_GPU_CASE_IDS = (
-    "wan2_2_i2v_a14b_2gpu",
-    "wan2_2_t2v_a14b_2gpu",
-    "wan2_2_t2v_a14b_teacache_2gpu",
-    "wan2_2_t2v_a14b_lora_2gpu",
-    "wan2_1_t2v_14b_2gpu",
-    "wan2_1_t2v_1.3b_cfg_parallel",
-    "fsdp-inference",
-    "mova_360p_tp2",
-    "mova_360p_ring1_uly2",
-    "mova_360p_ring2_uly1",
-    "ltx_2_two_stage_t2v",
-    "wan2_1_i2v_14b_480P_2gpu",
-    "wan2_1_i2v_14b_lora_2gpu",
-    "wan2_1_i2v_14b_720P_2gpu",
-    "qwen_image_t2i_2_gpus",
-    "zimage_image_t2i_2_gpus",
-    "zimage_image_t2i_2_gpus_non_square",
-    "flux_image_t2i_2_gpus",
-    "flux_2_image_t2i_2_gpus",
-)

-ACCURACY_ONE_GPU_CASES = _select_accuracy_cases(
-    ONE_GPU_CASES, ACCURACY_ONE_GPU_CASE_IDS
-)
-ACCURACY_TWO_GPU_CASES = _select_accuracy_cases(
-    TWO_GPU_CASES, ACCURACY_TWO_GPU_CASE_IDS
-)
+def _select_accuracy_cases(cases: list[DiffusionTestCase]) -> list[DiffusionTestCase]:
+    selected: list[DiffusionTestCase] = []
+    seen: dict[tuple, str] = {}
+    for case in cases:
+        if not case.run_component_accuracy_check:
+            continue
+
+        has_component_to_run = False
+        for component in ComponentType:
+            if should_skip_component(case, component):
+                continue
+
+            key = _component_accuracy_key(case, component)
+            representative = seen.get(key)
+            if representative is None:
+                seen[key] = case.id
+                has_component_to_run = True
+            else:
+                _COMPONENT_DUPLICATE_REASONS[(case.id, component)] = (
+                    f"{component.value} component already covered by {representative}"
+                )
+
+        if has_component_to_run:
+            selected.append(case)
+    return selected
+
+
+def get_component_duplicate_skip_reason(
+    case: DiffusionTestCase, component: ComponentType
+) -> str | None:
+    return _COMPONENT_DUPLICATE_REASONS.get((case.id, component))
+
+
+ACCURACY_ONE_GPU_CASES = _select_accuracy_cases(ONE_GPU_CASES)
+ACCURACY_TWO_GPU_CASES = _select_accuracy_cases(TWO_GPU_CASES)
@@ -2,6 +2,7 @@ from __future__ import annotations

 import json
 import os
+import shlex
 from contextlib import nullcontext
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Tuple
@@ -165,20 +166,24 @@ def resolve_component_path(


 def extract_component_path_overrides(extra_args: List[str]) -> Dict[str, str]:
+    normalized_args = []
+    for arg in extra_args:
+        normalized_args.extend(shlex.split(arg))
+
    component_paths: Dict[str, str] = {}
    index = 0
-    while index < len(extra_args):
-        arg = extra_args[index]
+    while index < len(normalized_args):
+        arg = normalized_args[index]
        key_part = arg.split("=", 1)[0] if "=" in arg else arg
        if key_part.startswith("--") and key_part.endswith("-path"):
            component = key_part[2:-5].replace("-", "_")
            if "=" in arg:
                component_paths[component] = arg.split("=", 1)[1]
-            elif index + 1 < len(extra_args) and not extra_args[index + 1].startswith(
-                "-"
-            ):
+            elif index + 1 < len(normalized_args) and not normalized_args[
+                index + 1
+            ].startswith("-"):
                index += 1
-                component_paths[component] = extra_args[index]
+                component_paths[component] = normalized_args[index]
        index += 1

    for component, path in component_paths.items():
@@ -228,12 +233,14 @@ def select_component_source(
        override_path = component_paths.get(key)
        if override_path is None:
            continue
-        assert has_component_files(override_path), (
+        resolved_override_path = maybe_download_model(override_path)
+        component_paths[key] = resolved_override_path
+        assert has_component_files(resolved_override_path), (
            f"Component override for {component.value} must point directly to a "
            f"component directory: {override_path}"
        )
        if component == ComponentType.TEXT_ENCODER:
-            assert is_text_encoder_config(override_path), (
+            assert is_text_encoder_config(resolved_override_path), (
                f"Text encoder override must point to a text encoder directory: "
                f"{override_path}"
            )
@@ -241,7 +248,7 @@ def select_component_source(
            base_model_id=model_id,
            base_model_root=base_model_root,
            component_paths=component_paths,
-            source_path=override_path,
+            source_path=resolved_override_path,
        )

    source_path = resolve_component_path(
@@ -161,6 +161,7 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
        DiffusionServerArgs(model_path=DEFAULT_JOYAI_IMAGE_EDIT_MODEL_NAME_FOR_TEST),
        TI2I_sampling_params,
        run_consistency_check=False,
+        run_component_accuracy_check=False,
    ),
    # Upscaling (Real-ESRGAN 4×) for T2I
    DiffusionTestCase(
@@ -344,6 +345,7 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
            },
        ),
        T2I_sampling_params,
+        run_component_accuracy_check=False,
    ),
 ]

@@ -538,6 +540,7 @@ TWO_GPU_CASES = [
            ],
        ),
        TI2V_sampling_params,
+        run_component_accuracy_check=False,
    ),
    DiffusionTestCase(
        "wan2_1_i2v_14b_480P_2gpu",
@@ -558,6 +561,7 @@ TWO_GPU_CASES = [
            ],
        ),
        T2V_sampling_params,
+        run_component_accuracy_check=False,
    ),
    # I2V LoRA test case
    DiffusionTestCase(
@@ -630,6 +634,7 @@ TWO_GPU_CASES = [
            ulysses_degree=2,
        ),
        TI2V_sampling_params,
+        run_component_accuracy_check=False,
    ),
 ]

@@ -2374,73 +2374,6 @@
            "expected_median_denoise_ms": 149.9,
            "estimated_full_test_time_s": 129.4
        },
-        "flux_2_nvfp4_t2i": {
-            "stages_ms": {
-                "InputValidationStage": 0.05,
-                "DecodingStage": 7.98,
-                "DenoisingStage": 7120.88,
-                "LatentPreparationStage": 0.63,
-                "TimestepPreparationStage": 23.91,
-                "TextEncodingStage": 463.52,
-                "ImageVAEEncodingStage": 0.01
-            },
-            "denoise_step_ms": {
-                "0": 95.82,
-                "1": 89.39,
-                "2": 109.92,
-                "3": 139.01,
-                "4": 143.6,
-                "5": 142.76,
-                "6": 139.62,
-                "7": 142.87,
-                "8": 140.81,
-                "9": 139.94,
-                "10": 141.82,
-                "11": 143.59,
-                "12": 146.37,
-                "13": 142.65,
-                "14": 140.02,
-                "15": 142.41,
-                "16": 139.28,
-                "17": 141.89,
-                "18": 139.87,
-                "19": 140.68,
-                "20": 144.58,
-                "21": 141.7,
-                "22": 141.1,
-                "23": 139.5,
-                "24": 144.49,
-                "25": 142.78,
-                "26": 141.11,
-                "27": 140.5,
-                "28": 140.88,
-                "29": 141.67,
-                "30": 140.75,
-                "31": 141.4,
-                "32": 142.53,
-                "33": 141.63,
-                "34": 142.45,
-                "35": 138.9,
-                "36": 141.22,
-                "37": 143.95,
-                "38": 141.33,
-                "39": 141.05,
-                "40": 139.45,
-                "41": 143.23,
-                "42": 142.17,
-                "43": 145.19,
-                "44": 142.61,
-                "45": 140.46,
-                "46": 142.39,
-                "47": 142.98,
-                "48": 141.62,
-                "49": 143.67
-            },
-            "expected_e2e_ms": 8029.29,
-            "expected_avg_denoise_ms": 140.06,
-            "expected_median_denoise_ms": 141.63,
-            "estimated_full_test_time_s": 128.2
-        },
        "ltx_2.3_one_stage_ti2v": {
            "stages_ms": {
                "InputValidationStage": 3.05,
@@ -7,6 +7,7 @@ from sglang.multimodal_gen.test.server.accuracy_config import (
 )
 from sglang.multimodal_gen.test.server.accuracy_testcase_configs import (
    ACCURACY_ONE_GPU_CASES,
+    get_component_duplicate_skip_reason,
 )
 from sglang.multimodal_gen.test.server.accuracy_utils import (
    run_native_component_accuracy_case,
@@ -22,6 +23,9 @@ class TestComponentAccuracy1GPU:
    def test_vae_accuracy(self, case):
        if should_skip_component(case, ComponentType.VAE):
            pytest.skip(get_skip_reason(case, ComponentType.VAE))
+        duplicate_reason = get_component_duplicate_skip_reason(case, ComponentType.VAE)
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
        run_native_component_accuracy_case(
            AccuracyEngine,
            case,
@@ -33,6 +37,11 @@ class TestComponentAccuracy1GPU:
    def test_transformer_accuracy(self, case):
        if should_skip_component(case, ComponentType.TRANSFORMER):
            pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER))
+        duplicate_reason = get_component_duplicate_skip_reason(
+            case, ComponentType.TRANSFORMER
+        )
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
        run_native_component_accuracy_case(
            AccuracyEngine,
            case,
@@ -44,6 +53,11 @@ class TestComponentAccuracy1GPU:
    def test_encoder_accuracy(self, case):
        if should_skip_component(case, ComponentType.TEXT_ENCODER):
            pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER))
+        duplicate_reason = get_component_duplicate_skip_reason(
+            case, ComponentType.TEXT_ENCODER
+        )
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
        run_text_encoder_accuracy_case(
            AccuracyEngine,
            case,
@@ -7,6 +7,7 @@ from sglang.multimodal_gen.test.server.accuracy_config import (
 )
 from sglang.multimodal_gen.test.server.accuracy_testcase_configs import (
    ACCURACY_TWO_GPU_CASES,
+    get_component_duplicate_skip_reason,
 )
 from sglang.multimodal_gen.test.server.accuracy_utils import (
    run_native_component_accuracy_case,
@@ -22,6 +23,9 @@ class TestComponentAccuracy2GPU:
    def test_vae_accuracy(self, case):
        if should_skip_component(case, ComponentType.VAE):
            pytest.skip(get_skip_reason(case, ComponentType.VAE))
+        duplicate_reason = get_component_duplicate_skip_reason(case, ComponentType.VAE)
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
        run_native_component_accuracy_case(
            AccuracyEngine,
            case,
@@ -33,6 +37,11 @@ class TestComponentAccuracy2GPU:
    def test_transformer_accuracy(self, case):
        if should_skip_component(case, ComponentType.TRANSFORMER):
            pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER))
+        duplicate_reason = get_component_duplicate_skip_reason(
+            case, ComponentType.TRANSFORMER
+        )
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
        run_native_component_accuracy_case(
            AccuracyEngine,
            case,
@@ -44,6 +53,11 @@ class TestComponentAccuracy2GPU:
    def test_encoder_accuracy(self, case):
        if should_skip_component(case, ComponentType.TEXT_ENCODER):
            pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER))
+        duplicate_reason = get_component_duplicate_skip_reason(
+            case, ComponentType.TEXT_ENCODER
+        )
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
        run_text_encoder_accuracy_case(
            AccuracyEngine,
            case,
@@ -257,6 +257,7 @@ class DiffusionTestCase:
    sampling_params: DiffusionSamplingParams
    run_perf_check: bool = True
    run_consistency_check: bool = True
+    run_component_accuracy_check: bool = True
    run_models_api_check: bool = True
    run_t2v_input_reference_check: bool = True
    run_lora_basic_api_check: bool = False
@@ -463,6 +464,7 @@ def _make_modelopt_ci_case(
        sampling_params,
        run_perf_check=False,
        run_consistency_check=False,
+        run_component_accuracy_check=False,
    )


@@ -39,10 +39,6 @@ SGL_TEST_FILES_CONSISTENCY_GT_BASE = SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE
 SGL_TEST_FILES_CONSISTENCY_GT_BASES = (
    SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE,
    SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE,
-    # Legacy fallback during migration from sglang-bot/sglang-ci-data
-    "https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/diffusion-ci/consistency_gt/official_generated",
-    "https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/diffusion-ci/consistency_gt/sglang_generated",
-    "https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/diffusion-ci/consistency_gt",
 )
 CONSISTENCY_THRESHOLD_JSON_PATH = (
    Path(__file__).resolve().parent / "server" / "consistency_threshold.json"