[diffusion] CI: switch CI data references to sgl-project/ci-data (#24299)

python/sglang/multimodal_gen/test/server/accuracy_config.py

@@ -40,7 +40,6 @@ DEFAULT_THRESHOLDS = {
 CASE_THRESHOLDS: Dict[str, Dict[ComponentType, float]] = {
     # Add overrides here when a specific model/component needs a different threshold.
     "flux_2_image_t2i": {ComponentType.TRANSFORMER: 0.99},
-    "flux_2_image_t2i_layerwise_offload": {ComponentType.TRANSFORMER: 0.99},
     "flux_2_image_t2i_2_gpus": {ComponentType.TRANSFORMER: 0.99},
     "flux_2_ti2i": {ComponentType.TRANSFORMER: 0.99},
     "flux_2_t2i_customized_vae_path": {ComponentType.TRANSFORMER: 0.99},
@@ -62,11 +61,6 @@ SKIP_COMPONENTS: Dict[str, Dict[ComponentType, ComponentSkip]] = {
             "Text encoder diverges from HF baseline despite 100% matched weights (CosSim ~0.47)"
         )
     },
-    "sana_image_t2i": {
-        ComponentType.VAE: ComponentSkip(
-            "HF AutoencoderDC checkpoint leaves required to_qkv_multiscale weights missing, so VAE transfer would compare against partially initialized reference weights"
-        )
-    },
     "qwen_image_t2i_cache_dit_enabled": {
         ComponentType.VAE: ComponentSkip(
             "Representative VAE accuracy is already covered by qwen_image_t2i for the same source component and topology"
@@ -104,6 +98,9 @@ SKIP_COMPONENTS: Dict[str, Dict[ComponentType, ComponentSkip]] = {
         ComponentType.VAE: ComponentSkip(
             "Representative VAE accuracy is already covered by zimage_image_t2i for the same source component and topology"
         ),
+        ComponentType.TRANSFORMER: ComponentSkip(
+            "FP8 transformer override cannot be materialized by the Diffusers reference loader"
+        ),
         ComponentType.TEXT_ENCODER: ComponentSkip(
             "Representative text encoder accuracy is already covered by zimage_image_t2i for the same source component and topology"
         ),

python/sglang/multimodal_gen/test/server/accuracy_testcase_configs.py

@@ -1,5 +1,13 @@
 from __future__ import annotations
 
+from sglang.multimodal_gen.test.server.accuracy_config import (
+    ComponentType,
+    should_skip_component,
+)
+from sglang.multimodal_gen.test.server.accuracy_utils import (
+    extract_component_path_overrides,
+)
+from sglang.multimodal_gen.test.server.component_accuracy import COMPONENT_SPECS
 from sglang.multimodal_gen.test.server.gpu_cases import (
     ONE_GPU_CASES,
     TWO_GPU_CASES,
@@ -7,70 +15,62 @@ from sglang.multimodal_gen.test.server.gpu_cases import (
 from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase
 
 
-def _select_accuracy_cases(
+def _component_accuracy_key(case: DiffusionTestCase, component: ComponentType) -> tuple:
-    cases: list[DiffusionTestCase], enabled_ids: tuple[str, ...]
+    server_args = case.server_args
-) -> list[DiffusionTestCase]:
+    component_paths = extract_component_path_overrides(server_args.extras)
-    enabled = set(enabled_ids)
+    override_path = None
-    return [case for case in cases if case.id in enabled]
+    for key in (component.value, *COMPONENT_SPECS[component].model_index_keys):
+        if key in component_paths:
+            override_path = component_paths[key]
+            break
+
+    return (
+        component.value,
+        server_args.model_path,
+        override_path,
+        server_args.num_gpus,
+        server_args.tp_size,
+        server_args.ulysses_degree,
+        server_args.ring_degree,
+        server_args.cfg_parallel,
+    )
 
 
-ACCURACY_ONE_GPU_CASE_IDS = (
+_COMPONENT_DUPLICATE_REASONS: dict[tuple[str, ComponentType], str] = {}
-    "qwen_image_t2i",
-    "qwen_image_t2i_cache_dit_enabled",
-    "flux_image_t2i",
-    "flux_2_image_t2i",
-    "flux_2_klein_image_t2i",
-    "layerwise_offload",
-    "zimage_image_t2i",
-    "zimage_image_t2i_fp8",
-    "zimage_image_t2i_multi_lora",
-    "qwen_image_edit_ti2i",
-    "qwen_image_edit_2509_ti2i",
-    "qwen_image_edit_2511_ti2i",
-    "qwen_image_layered_i2i",
-    "flux_2_image_t2i_upscaling_4x",
-    "mova_360p_1gpu",
-    "wan2_1_t2v_1.3b",
-    "wan2_1_t2v_1.3b_teacache_enabled",
-    "wan2_1_t2v_1.3b_frame_interp_2x",
-    "wan2_1_t2v_1.3b_upscaling_4x",
-    "wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x",
-    "wan2_1_t2v_1_3b_lora_1gpu",
-    "flux_2_ti2i",
-    "flux_2_t2i_customized_vae_path",
-    "fast_hunyuan_video",
-    "wan2_2_ti2v_5b",
-    "fastwan2_2_ti2v_5b",
-    "hunyuan3d_shape_gen",
-    "turbo_wan2_1_t2v_1.3b",
-    "flux_2_ti2i_multi_image_cache_dit",
-)
 
-ACCURACY_TWO_GPU_CASE_IDS = (
-    "wan2_2_i2v_a14b_2gpu",
-    "wan2_2_t2v_a14b_2gpu",
-    "wan2_2_t2v_a14b_teacache_2gpu",
-    "wan2_2_t2v_a14b_lora_2gpu",
-    "wan2_1_t2v_14b_2gpu",
-    "wan2_1_t2v_1.3b_cfg_parallel",
-    "fsdp-inference",
-    "mova_360p_tp2",
-    "mova_360p_ring1_uly2",
-    "mova_360p_ring2_uly1",
-    "ltx_2_two_stage_t2v",
-    "wan2_1_i2v_14b_480P_2gpu",
-    "wan2_1_i2v_14b_lora_2gpu",
-    "wan2_1_i2v_14b_720P_2gpu",
-    "qwen_image_t2i_2_gpus",
-    "zimage_image_t2i_2_gpus",
-    "zimage_image_t2i_2_gpus_non_square",
-    "flux_image_t2i_2_gpus",
-    "flux_2_image_t2i_2_gpus",
-)
 
-ACCURACY_ONE_GPU_CASES = _select_accuracy_cases(
+def _select_accuracy_cases(cases: list[DiffusionTestCase]) -> list[DiffusionTestCase]:
-    ONE_GPU_CASES, ACCURACY_ONE_GPU_CASE_IDS
+    selected: list[DiffusionTestCase] = []
-)
+    seen: dict[tuple, str] = {}
-ACCURACY_TWO_GPU_CASES = _select_accuracy_cases(
+    for case in cases:
-    TWO_GPU_CASES, ACCURACY_TWO_GPU_CASE_IDS
+        if not case.run_component_accuracy_check:
-)
+            continue
+
+        has_component_to_run = False
+        for component in ComponentType:
+            if should_skip_component(case, component):
+                continue
+
+            key = _component_accuracy_key(case, component)
+            representative = seen.get(key)
+            if representative is None:
+                seen[key] = case.id
+                has_component_to_run = True
+            else:
+                _COMPONENT_DUPLICATE_REASONS[(case.id, component)] = (
+                    f"{component.value} component already covered by {representative}"
+                )
+
+        if has_component_to_run:
+            selected.append(case)
+    return selected
+
+
+def get_component_duplicate_skip_reason(
+    case: DiffusionTestCase, component: ComponentType
+) -> str | None:
+    return _COMPONENT_DUPLICATE_REASONS.get((case.id, component))
+
+
+ACCURACY_ONE_GPU_CASES = _select_accuracy_cases(ONE_GPU_CASES)
+ACCURACY_TWO_GPU_CASES = _select_accuracy_cases(TWO_GPU_CASES)

python/sglang/multimodal_gen/test/server/accuracy_utils.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 import os
+import shlex
 from contextlib import nullcontext
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Tuple
@@ -165,20 +166,24 @@ def resolve_component_path(
 
 
 def extract_component_path_overrides(extra_args: List[str]) -> Dict[str, str]:
+    normalized_args = []
+    for arg in extra_args:
+        normalized_args.extend(shlex.split(arg))
+
     component_paths: Dict[str, str] = {}
     index = 0
-    while index < len(extra_args):
+    while index < len(normalized_args):
-        arg = extra_args[index]
+        arg = normalized_args[index]
         key_part = arg.split("=", 1)[0] if "=" in arg else arg
         if key_part.startswith("--") and key_part.endswith("-path"):
             component = key_part[2:-5].replace("-", "_")
             if "=" in arg:
                 component_paths[component] = arg.split("=", 1)[1]
-            elif index + 1 < len(extra_args) and not extra_args[index + 1].startswith(
+            elif index + 1 < len(normalized_args) and not normalized_args[
-                "-"
+                index + 1
-            ):
+            ].startswith("-"):
                 index += 1
-                component_paths[component] = extra_args[index]
+                component_paths[component] = normalized_args[index]
         index += 1
 
     for component, path in component_paths.items():
@@ -228,12 +233,14 @@ def select_component_source(
         override_path = component_paths.get(key)
         if override_path is None:
             continue
-        assert has_component_files(override_path), (
+        resolved_override_path = maybe_download_model(override_path)
+        component_paths[key] = resolved_override_path
+        assert has_component_files(resolved_override_path), (
             f"Component override for {component.value} must point directly to a "
             f"component directory: {override_path}"
         )
         if component == ComponentType.TEXT_ENCODER:
-            assert is_text_encoder_config(override_path), (
+            assert is_text_encoder_config(resolved_override_path), (
                 f"Text encoder override must point to a text encoder directory: "
                 f"{override_path}"
             )
@@ -241,7 +248,7 @@ def select_component_source(
             base_model_id=model_id,
             base_model_root=base_model_root,
             component_paths=component_paths,
-            source_path=override_path,
+            source_path=resolved_override_path,
         )
 
     source_path = resolve_component_path(

python/sglang/multimodal_gen/test/server/gpu_cases.py

@@ -161,6 +161,7 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
         DiffusionServerArgs(model_path=DEFAULT_JOYAI_IMAGE_EDIT_MODEL_NAME_FOR_TEST),
         TI2I_sampling_params,
         run_consistency_check=False,
+        run_component_accuracy_check=False,
     ),
     # Upscaling (Real-ESRGAN 4×) for T2I
     DiffusionTestCase(
@@ -344,6 +345,7 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
             },
         ),
         T2I_sampling_params,
+        run_component_accuracy_check=False,
     ),
 ]
 
@@ -538,6 +540,7 @@ TWO_GPU_CASES = [
             ],
         ),
         TI2V_sampling_params,
+        run_component_accuracy_check=False,
     ),
     DiffusionTestCase(
         "wan2_1_i2v_14b_480P_2gpu",
@@ -558,6 +561,7 @@ TWO_GPU_CASES = [
             ],
         ),
         T2V_sampling_params,
+        run_component_accuracy_check=False,
     ),
     # I2V LoRA test case
     DiffusionTestCase(
@@ -630,6 +634,7 @@ TWO_GPU_CASES = [
             ulysses_degree=2,
         ),
         TI2V_sampling_params,
+        run_component_accuracy_check=False,
     ),
 ]
 

python/sglang/multimodal_gen/test/server/perf_baselines.json

@@ -2374,73 +2374,6 @@
             "expected_median_denoise_ms": 149.9,
             "estimated_full_test_time_s": 129.4
         },
-        "flux_2_nvfp4_t2i": {
-            "stages_ms": {
-                "InputValidationStage": 0.05,
-                "DecodingStage": 7.98,
-                "DenoisingStage": 7120.88,
-                "LatentPreparationStage": 0.63,
-                "TimestepPreparationStage": 23.91,
-                "TextEncodingStage": 463.52,
-                "ImageVAEEncodingStage": 0.01
-            },
-            "denoise_step_ms": {
-                "0": 95.82,
-                "1": 89.39,
-                "2": 109.92,
-                "3": 139.01,
-                "4": 143.6,
-                "5": 142.76,
-                "6": 139.62,
-                "7": 142.87,
-                "8": 140.81,
-                "9": 139.94,
-                "10": 141.82,
-                "11": 143.59,
-                "12": 146.37,
-                "13": 142.65,
-                "14": 140.02,
-                "15": 142.41,
-                "16": 139.28,
-                "17": 141.89,
-                "18": 139.87,
-                "19": 140.68,
-                "20": 144.58,
-                "21": 141.7,
-                "22": 141.1,
-                "23": 139.5,
-                "24": 144.49,
-                "25": 142.78,
-                "26": 141.11,
-                "27": 140.5,
-                "28": 140.88,
-                "29": 141.67,
-                "30": 140.75,
-                "31": 141.4,
-                "32": 142.53,
-                "33": 141.63,
-                "34": 142.45,
-                "35": 138.9,
-                "36": 141.22,
-                "37": 143.95,
-                "38": 141.33,
-                "39": 141.05,
-                "40": 139.45,
-                "41": 143.23,
-                "42": 142.17,
-                "43": 145.19,
-                "44": 142.61,
-                "45": 140.46,
-                "46": 142.39,
-                "47": 142.98,
-                "48": 141.62,
-                "49": 143.67
-            },
-            "expected_e2e_ms": 8029.29,
-            "expected_avg_denoise_ms": 140.06,
-            "expected_median_denoise_ms": 141.63,
-            "estimated_full_test_time_s": 128.2
-        },
         "ltx_2.3_one_stage_ti2v": {
             "stages_ms": {
                 "InputValidationStage": 3.05,

python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py

@@ -7,6 +7,7 @@ from sglang.multimodal_gen.test.server.accuracy_config import (
 )
 from sglang.multimodal_gen.test.server.accuracy_testcase_configs import (
     ACCURACY_ONE_GPU_CASES,
+    get_component_duplicate_skip_reason,
 )
 from sglang.multimodal_gen.test.server.accuracy_utils import (
     run_native_component_accuracy_case,
@@ -22,6 +23,9 @@ class TestComponentAccuracy1GPU:
     def test_vae_accuracy(self, case):
         if should_skip_component(case, ComponentType.VAE):
             pytest.skip(get_skip_reason(case, ComponentType.VAE))
+        duplicate_reason = get_component_duplicate_skip_reason(case, ComponentType.VAE)
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
         run_native_component_accuracy_case(
             AccuracyEngine,
             case,
@@ -33,6 +37,11 @@ class TestComponentAccuracy1GPU:
     def test_transformer_accuracy(self, case):
         if should_skip_component(case, ComponentType.TRANSFORMER):
             pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER))
+        duplicate_reason = get_component_duplicate_skip_reason(
+            case, ComponentType.TRANSFORMER
+        )
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
         run_native_component_accuracy_case(
             AccuracyEngine,
             case,
@@ -44,6 +53,11 @@ class TestComponentAccuracy1GPU:
     def test_encoder_accuracy(self, case):
         if should_skip_component(case, ComponentType.TEXT_ENCODER):
             pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER))
+        duplicate_reason = get_component_duplicate_skip_reason(
+            case, ComponentType.TEXT_ENCODER
+        )
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
         run_text_encoder_accuracy_case(
             AccuracyEngine,
             case,

python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py

@@ -7,6 +7,7 @@ from sglang.multimodal_gen.test.server.accuracy_config import (
 )
 from sglang.multimodal_gen.test.server.accuracy_testcase_configs import (
     ACCURACY_TWO_GPU_CASES,
+    get_component_duplicate_skip_reason,
 )
 from sglang.multimodal_gen.test.server.accuracy_utils import (
     run_native_component_accuracy_case,
@@ -22,6 +23,9 @@ class TestComponentAccuracy2GPU:
     def test_vae_accuracy(self, case):
         if should_skip_component(case, ComponentType.VAE):
             pytest.skip(get_skip_reason(case, ComponentType.VAE))
+        duplicate_reason = get_component_duplicate_skip_reason(case, ComponentType.VAE)
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
         run_native_component_accuracy_case(
             AccuracyEngine,
             case,
@@ -33,6 +37,11 @@ class TestComponentAccuracy2GPU:
     def test_transformer_accuracy(self, case):
         if should_skip_component(case, ComponentType.TRANSFORMER):
             pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER))
+        duplicate_reason = get_component_duplicate_skip_reason(
+            case, ComponentType.TRANSFORMER
+        )
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
         run_native_component_accuracy_case(
             AccuracyEngine,
             case,
@@ -44,6 +53,11 @@ class TestComponentAccuracy2GPU:
     def test_encoder_accuracy(self, case):
         if should_skip_component(case, ComponentType.TEXT_ENCODER):
             pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER))
+        duplicate_reason = get_component_duplicate_skip_reason(
+            case, ComponentType.TEXT_ENCODER
+        )
+        if duplicate_reason:
+            pytest.skip(duplicate_reason)
         run_text_encoder_accuracy_case(
             AccuracyEngine,
             case,

python/sglang/multimodal_gen/test/server/testcase_configs.py

@@ -257,6 +257,7 @@ class DiffusionTestCase:
     sampling_params: DiffusionSamplingParams
     run_perf_check: bool = True
     run_consistency_check: bool = True
+    run_component_accuracy_check: bool = True
     run_models_api_check: bool = True
     run_t2v_input_reference_check: bool = True
     run_lora_basic_api_check: bool = False
@@ -463,6 +464,7 @@ def _make_modelopt_ci_case(
         sampling_params,
         run_perf_check=False,
         run_consistency_check=False,
+        run_component_accuracy_check=False,
     )
 
 

python/sglang/multimodal_gen/test/test_utils.py

@@ -39,10 +39,6 @@ SGL_TEST_FILES_CONSISTENCY_GT_BASE = SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE
 SGL_TEST_FILES_CONSISTENCY_GT_BASES = (
     SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE,
     SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE,
-    # Legacy fallback during migration from sglang-bot/sglang-ci-data
-    "https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/diffusion-ci/consistency_gt/official_generated",
-    "https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/diffusion-ci/consistency_gt/sglang_generated",
-    "https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/diffusion-ci/consistency_gt",
 )
 CONSISTENCY_THRESHOLD_JSON_PATH = (
     Path(__file__).resolve().parent / "server" / "consistency_threshold.json"

scripts/ci/utils/diffusion/generate_diffusion_dashboard.py

@@ -1,6 +1,6 @@
 """Generate a Markdown dashboard for diffusion cross-framework comparisons.
 
-Reads current comparison results + historical data from sglang-ci-data repo
+Reads current comparison results + historical data from sgl-project/ci-data repo
 and produces a Markdown report with tables and trend charts saved as PNG files.
 
 Usage:
@@ -19,16 +19,16 @@ import sys
 from datetime import datetime, timezone
 
 # ---------------------------------------------------------------------------
-# History fetching (from sglang-ci-data repo via GitHub API)
+# History fetching (from sgl-project/ci-data repo via GitHub API)
 # ---------------------------------------------------------------------------
 
-CI_DATA_REPO_OWNER = "sglang-bot"
+CI_DATA_REPO_OWNER = "sgl-project"
-CI_DATA_REPO_NAME = "sglang-ci-data"
+CI_DATA_REPO_NAME = "ci-data"
 CI_DATA_BRANCH = "main"
 HISTORY_PREFIX = "diffusion-comparisons"
 MAX_HISTORY_RUNS = 14
 
-# Base URL for chart images pushed to sglang-ci-data
+# Base URL for chart images pushed to sgl-project/ci-data
 CHARTS_RAW_BASE_URL = (
     f"https://raw.githubusercontent.com/{CI_DATA_REPO_OWNER}/{CI_DATA_REPO_NAME}"
     f"/{CI_DATA_BRANCH}/{HISTORY_PREFIX}/charts"
@@ -58,7 +58,7 @@ def _github_get(url: str, token: str) -> dict | list | None:
 
 
 def fetch_history_from_github(token: str) -> list[dict]:
-    """Fetch recent comparison result JSONs from sglang-ci-data repo."""
+    """Fetch recent comparison result JSONs from sgl-project/ci-data repo."""
     print("Fetching historical comparison data from GitHub...")
     url = (
         f"https://api.github.com/repos/{CI_DATA_REPO_OWNER}/{CI_DATA_REPO_NAME}"
@@ -775,7 +775,7 @@ def main():
     parser.add_argument(
         "--fetch-history",
         action="store_true",
-        help="Fetch history from sglang-ci-data GitHub repo",
+        help="Fetch history from ci-data GitHub repo",
     )
     parser.add_argument(
         "--step-summary",

scripts/ci/utils/diffusion/publish_comparison_results.py

@@ -1,4 +1,4 @@
-"""Publish diffusion comparison results to sglang-bot/sglang-ci-data repo.
+"""Publish diffusion comparison results to sgl-project/ci-data repo.
 
 Pushes comparison-results.json, dashboard.md, and chart PNG files to the
 ci-data repository for historical tracking. Chart PNGs are stored under
@@ -49,8 +49,8 @@ else:
     )
 
 # Repository configuration
-REPO_OWNER = "sglang-bot"
+REPO_OWNER = "sgl-project"
-REPO_NAME = "sglang-ci-data"
+REPO_NAME = "ci-data"
 BRANCH = "main"
 STORAGE_PREFIX = "diffusion-comparisons"
 
@@ -196,7 +196,7 @@ def publish_comparison(
 
 def main():
     parser = argparse.ArgumentParser(
-        description="Publish diffusion comparison results to sglang-ci-data"
+        description="Publish diffusion comparison results to ci-data"
     )
     parser.add_argument(
         "--results",

scripts/ci/utils/publish_traces.py

@@ -352,8 +352,8 @@ def publish_traces_from_files(files_to_upload, run_id, run_number):
         sys.exit(1)
 
     # Repository configuration
-    repo_owner = "sglang-bot"
+    repo_owner = "sgl-project"
-    repo_name = "sglang-ci-data"
+    repo_name = "ci-data"
     branch = "main"
 
     # Verify token permissions before proceeding