diff --git a/python/sglang/multimodal_gen/test/server/accuracy_config.py b/python/sglang/multimodal_gen/test/server/accuracy_config.py index 46b20a23a..5eecb88d1 100644 --- a/python/sglang/multimodal_gen/test/server/accuracy_config.py +++ b/python/sglang/multimodal_gen/test/server/accuracy_config.py @@ -40,7 +40,6 @@ DEFAULT_THRESHOLDS = { CASE_THRESHOLDS: Dict[str, Dict[ComponentType, float]] = { # Add overrides here when a specific model/component needs a different threshold. "flux_2_image_t2i": {ComponentType.TRANSFORMER: 0.99}, - "flux_2_image_t2i_layerwise_offload": {ComponentType.TRANSFORMER: 0.99}, "flux_2_image_t2i_2_gpus": {ComponentType.TRANSFORMER: 0.99}, "flux_2_ti2i": {ComponentType.TRANSFORMER: 0.99}, "flux_2_t2i_customized_vae_path": {ComponentType.TRANSFORMER: 0.99}, @@ -62,11 +61,6 @@ SKIP_COMPONENTS: Dict[str, Dict[ComponentType, ComponentSkip]] = { "Text encoder diverges from HF baseline despite 100% matched weights (CosSim ~0.47)" ) }, - "sana_image_t2i": { - ComponentType.VAE: ComponentSkip( - "HF AutoencoderDC checkpoint leaves required to_qkv_multiscale weights missing, so VAE transfer would compare against partially initialized reference weights" - ) - }, "qwen_image_t2i_cache_dit_enabled": { ComponentType.VAE: ComponentSkip( "Representative VAE accuracy is already covered by qwen_image_t2i for the same source component and topology" @@ -104,6 +98,9 @@ SKIP_COMPONENTS: Dict[str, Dict[ComponentType, ComponentSkip]] = { ComponentType.VAE: ComponentSkip( "Representative VAE accuracy is already covered by zimage_image_t2i for the same source component and topology" ), + ComponentType.TRANSFORMER: ComponentSkip( + "FP8 transformer override cannot be materialized by the Diffusers reference loader" + ), ComponentType.TEXT_ENCODER: ComponentSkip( "Representative text encoder accuracy is already covered by zimage_image_t2i for the same source component and topology" ), diff --git a/python/sglang/multimodal_gen/test/server/accuracy_testcase_configs.py b/python/sglang/multimodal_gen/test/server/accuracy_testcase_configs.py index ec8654b3a..ffe91ffa0 100644 --- a/python/sglang/multimodal_gen/test/server/accuracy_testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/accuracy_testcase_configs.py @@ -1,5 +1,13 @@ from __future__ import annotations +from sglang.multimodal_gen.test.server.accuracy_config import ( + ComponentType, + should_skip_component, +) +from sglang.multimodal_gen.test.server.accuracy_utils import ( + extract_component_path_overrides, +) +from sglang.multimodal_gen.test.server.component_accuracy import COMPONENT_SPECS from sglang.multimodal_gen.test.server.gpu_cases import ( ONE_GPU_CASES, TWO_GPU_CASES, @@ -7,70 +15,62 @@ from sglang.multimodal_gen.test.server.gpu_cases import ( from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase -def _select_accuracy_cases( - cases: list[DiffusionTestCase], enabled_ids: tuple[str, ...] -) -> list[DiffusionTestCase]: - enabled = set(enabled_ids) - return [case for case in cases if case.id in enabled] +def _component_accuracy_key(case: DiffusionTestCase, component: ComponentType) -> tuple: + server_args = case.server_args + component_paths = extract_component_path_overrides(server_args.extras) + override_path = None + for key in (component.value, *COMPONENT_SPECS[component].model_index_keys): + if key in component_paths: + override_path = component_paths[key] + break + + return ( + component.value, + server_args.model_path, + override_path, + server_args.num_gpus, + server_args.tp_size, + server_args.ulysses_degree, + server_args.ring_degree, + server_args.cfg_parallel, + ) -ACCURACY_ONE_GPU_CASE_IDS = ( - "qwen_image_t2i", - "qwen_image_t2i_cache_dit_enabled", - "flux_image_t2i", - "flux_2_image_t2i", - "flux_2_klein_image_t2i", - "layerwise_offload", - "zimage_image_t2i", - "zimage_image_t2i_fp8", - "zimage_image_t2i_multi_lora", - "qwen_image_edit_ti2i", - "qwen_image_edit_2509_ti2i", - "qwen_image_edit_2511_ti2i", - "qwen_image_layered_i2i", - "flux_2_image_t2i_upscaling_4x", - "mova_360p_1gpu", - "wan2_1_t2v_1.3b", - "wan2_1_t2v_1.3b_teacache_enabled", - "wan2_1_t2v_1.3b_frame_interp_2x", - "wan2_1_t2v_1.3b_upscaling_4x", - "wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x", - "wan2_1_t2v_1_3b_lora_1gpu", - "flux_2_ti2i", - "flux_2_t2i_customized_vae_path", - "fast_hunyuan_video", - "wan2_2_ti2v_5b", - "fastwan2_2_ti2v_5b", - "hunyuan3d_shape_gen", - "turbo_wan2_1_t2v_1.3b", - "flux_2_ti2i_multi_image_cache_dit", -) +_COMPONENT_DUPLICATE_REASONS: dict[tuple[str, ComponentType], str] = {} -ACCURACY_TWO_GPU_CASE_IDS = ( - "wan2_2_i2v_a14b_2gpu", - "wan2_2_t2v_a14b_2gpu", - "wan2_2_t2v_a14b_teacache_2gpu", - "wan2_2_t2v_a14b_lora_2gpu", - "wan2_1_t2v_14b_2gpu", - "wan2_1_t2v_1.3b_cfg_parallel", - "fsdp-inference", - "mova_360p_tp2", - "mova_360p_ring1_uly2", - "mova_360p_ring2_uly1", - "ltx_2_two_stage_t2v", - "wan2_1_i2v_14b_480P_2gpu", - "wan2_1_i2v_14b_lora_2gpu", - "wan2_1_i2v_14b_720P_2gpu", - "qwen_image_t2i_2_gpus", - "zimage_image_t2i_2_gpus", - "zimage_image_t2i_2_gpus_non_square", - "flux_image_t2i_2_gpus", - "flux_2_image_t2i_2_gpus", -) -ACCURACY_ONE_GPU_CASES = _select_accuracy_cases( - ONE_GPU_CASES, ACCURACY_ONE_GPU_CASE_IDS -) -ACCURACY_TWO_GPU_CASES = _select_accuracy_cases( - TWO_GPU_CASES, ACCURACY_TWO_GPU_CASE_IDS -) +def _select_accuracy_cases(cases: list[DiffusionTestCase]) -> list[DiffusionTestCase]: + selected: list[DiffusionTestCase] = [] + seen: dict[tuple, str] = {} + for case in cases: + if not case.run_component_accuracy_check: + continue + + has_component_to_run = False + for component in ComponentType: + if should_skip_component(case, component): + continue + + key = _component_accuracy_key(case, component) + representative = seen.get(key) + if representative is None: + seen[key] = case.id + has_component_to_run = True + else: + _COMPONENT_DUPLICATE_REASONS[(case.id, component)] = ( + f"{component.value} component already covered by {representative}" + ) + + if has_component_to_run: + selected.append(case) + return selected + + +def get_component_duplicate_skip_reason( + case: DiffusionTestCase, component: ComponentType +) -> str | None: + return _COMPONENT_DUPLICATE_REASONS.get((case.id, component)) + + +ACCURACY_ONE_GPU_CASES = _select_accuracy_cases(ONE_GPU_CASES) +ACCURACY_TWO_GPU_CASES = _select_accuracy_cases(TWO_GPU_CASES) diff --git a/python/sglang/multimodal_gen/test/server/accuracy_utils.py b/python/sglang/multimodal_gen/test/server/accuracy_utils.py index 943fb4e34..6da2d2c75 100644 --- a/python/sglang/multimodal_gen/test/server/accuracy_utils.py +++ b/python/sglang/multimodal_gen/test/server/accuracy_utils.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import os +import shlex from contextlib import nullcontext from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple @@ -165,20 +166,24 @@ def resolve_component_path( def extract_component_path_overrides(extra_args: List[str]) -> Dict[str, str]: + normalized_args = [] + for arg in extra_args: + normalized_args.extend(shlex.split(arg)) + component_paths: Dict[str, str] = {} index = 0 - while index < len(extra_args): - arg = extra_args[index] + while index < len(normalized_args): + arg = normalized_args[index] key_part = arg.split("=", 1)[0] if "=" in arg else arg if key_part.startswith("--") and key_part.endswith("-path"): component = key_part[2:-5].replace("-", "_") if "=" in arg: component_paths[component] = arg.split("=", 1)[1] - elif index + 1 < len(extra_args) and not extra_args[index + 1].startswith( - "-" - ): + elif index + 1 < len(normalized_args) and not normalized_args[ + index + 1 + ].startswith("-"): index += 1 - component_paths[component] = extra_args[index] + component_paths[component] = normalized_args[index] index += 1 for component, path in component_paths.items(): @@ -228,12 +233,14 @@ def select_component_source( override_path = component_paths.get(key) if override_path is None: continue - assert has_component_files(override_path), ( + resolved_override_path = maybe_download_model(override_path) + component_paths[key] = resolved_override_path + assert has_component_files(resolved_override_path), ( f"Component override for {component.value} must point directly to a " f"component directory: {override_path}" ) if component == ComponentType.TEXT_ENCODER: - assert is_text_encoder_config(override_path), ( + assert is_text_encoder_config(resolved_override_path), ( f"Text encoder override must point to a text encoder directory: " f"{override_path}" ) @@ -241,7 +248,7 @@ def select_component_source( base_model_id=model_id, base_model_root=base_model_root, component_paths=component_paths, - source_path=override_path, + source_path=resolved_override_path, ) source_path = resolve_component_path( diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index 31332ca04..87f239327 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -161,6 +161,7 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [ DiffusionServerArgs(model_path=DEFAULT_JOYAI_IMAGE_EDIT_MODEL_NAME_FOR_TEST), TI2I_sampling_params, run_consistency_check=False, + run_component_accuracy_check=False, ), # Upscaling (Real-ESRGAN 4×) for T2I DiffusionTestCase( @@ -344,6 +345,7 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [ }, ), T2I_sampling_params, + run_component_accuracy_check=False, ), ] @@ -538,6 +540,7 @@ TWO_GPU_CASES = [ ], ), TI2V_sampling_params, + run_component_accuracy_check=False, ), DiffusionTestCase( "wan2_1_i2v_14b_480P_2gpu", @@ -558,6 +561,7 @@ TWO_GPU_CASES = [ ], ), T2V_sampling_params, + run_component_accuracy_check=False, ), # I2V LoRA test case DiffusionTestCase( @@ -630,6 +634,7 @@ TWO_GPU_CASES = [ ulysses_degree=2, ), TI2V_sampling_params, + run_component_accuracy_check=False, ), ] diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines.json b/python/sglang/multimodal_gen/test/server/perf_baselines.json index cbf0a5a52..7d5a7e96a 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines.json @@ -2374,73 +2374,6 @@ "expected_median_denoise_ms": 149.9, "estimated_full_test_time_s": 129.4 }, - "flux_2_nvfp4_t2i": { - "stages_ms": { - "InputValidationStage": 0.05, - "DecodingStage": 7.98, - "DenoisingStage": 7120.88, - "LatentPreparationStage": 0.63, - "TimestepPreparationStage": 23.91, - "TextEncodingStage": 463.52, - "ImageVAEEncodingStage": 0.01 - }, - "denoise_step_ms": { - "0": 95.82, - "1": 89.39, - "2": 109.92, - "3": 139.01, - "4": 143.6, - "5": 142.76, - "6": 139.62, - "7": 142.87, - "8": 140.81, - "9": 139.94, - "10": 141.82, - "11": 143.59, - "12": 146.37, - "13": 142.65, - "14": 140.02, - "15": 142.41, - "16": 139.28, - "17": 141.89, - "18": 139.87, - "19": 140.68, - "20": 144.58, - "21": 141.7, - "22": 141.1, - "23": 139.5, - "24": 144.49, - "25": 142.78, - "26": 141.11, - "27": 140.5, - "28": 140.88, - "29": 141.67, - "30": 140.75, - "31": 141.4, - "32": 142.53, - "33": 141.63, - "34": 142.45, - "35": 138.9, - "36": 141.22, - "37": 143.95, - "38": 141.33, - "39": 141.05, - "40": 139.45, - "41": 143.23, - "42": 142.17, - "43": 145.19, - "44": 142.61, - "45": 140.46, - "46": 142.39, - "47": 142.98, - "48": 141.62, - "49": 143.67 - }, - "expected_e2e_ms": 8029.29, - "expected_avg_denoise_ms": 140.06, - "expected_median_denoise_ms": 141.63, - "estimated_full_test_time_s": 128.2 - }, "ltx_2.3_one_stage_ti2v": { "stages_ms": { "InputValidationStage": 3.05, diff --git a/python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py b/python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py index 12d606d8a..1d41f9b0b 100644 --- a/python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py +++ b/python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py @@ -7,6 +7,7 @@ from sglang.multimodal_gen.test.server.accuracy_config import ( ) from sglang.multimodal_gen.test.server.accuracy_testcase_configs import ( ACCURACY_ONE_GPU_CASES, + get_component_duplicate_skip_reason, ) from sglang.multimodal_gen.test.server.accuracy_utils import ( run_native_component_accuracy_case, @@ -22,6 +23,9 @@ class TestComponentAccuracy1GPU: def test_vae_accuracy(self, case): if should_skip_component(case, ComponentType.VAE): pytest.skip(get_skip_reason(case, ComponentType.VAE)) + duplicate_reason = get_component_duplicate_skip_reason(case, ComponentType.VAE) + if duplicate_reason: + pytest.skip(duplicate_reason) run_native_component_accuracy_case( AccuracyEngine, case, @@ -33,6 +37,11 @@ class TestComponentAccuracy1GPU: def test_transformer_accuracy(self, case): if should_skip_component(case, ComponentType.TRANSFORMER): pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER)) + duplicate_reason = get_component_duplicate_skip_reason( + case, ComponentType.TRANSFORMER + ) + if duplicate_reason: + pytest.skip(duplicate_reason) run_native_component_accuracy_case( AccuracyEngine, case, @@ -44,6 +53,11 @@ class TestComponentAccuracy1GPU: def test_encoder_accuracy(self, case): if should_skip_component(case, ComponentType.TEXT_ENCODER): pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER)) + duplicate_reason = get_component_duplicate_skip_reason( + case, ComponentType.TEXT_ENCODER + ) + if duplicate_reason: + pytest.skip(duplicate_reason) run_text_encoder_accuracy_case( AccuracyEngine, case, diff --git a/python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py b/python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py index af85fb65f..8e8d3dfaa 100644 --- a/python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py +++ b/python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py @@ -7,6 +7,7 @@ from sglang.multimodal_gen.test.server.accuracy_config import ( ) from sglang.multimodal_gen.test.server.accuracy_testcase_configs import ( ACCURACY_TWO_GPU_CASES, + get_component_duplicate_skip_reason, ) from sglang.multimodal_gen.test.server.accuracy_utils import ( run_native_component_accuracy_case, @@ -22,6 +23,9 @@ class TestComponentAccuracy2GPU: def test_vae_accuracy(self, case): if should_skip_component(case, ComponentType.VAE): pytest.skip(get_skip_reason(case, ComponentType.VAE)) + duplicate_reason = get_component_duplicate_skip_reason(case, ComponentType.VAE) + if duplicate_reason: + pytest.skip(duplicate_reason) run_native_component_accuracy_case( AccuracyEngine, case, @@ -33,6 +37,11 @@ class TestComponentAccuracy2GPU: def test_transformer_accuracy(self, case): if should_skip_component(case, ComponentType.TRANSFORMER): pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER)) + duplicate_reason = get_component_duplicate_skip_reason( + case, ComponentType.TRANSFORMER + ) + if duplicate_reason: + pytest.skip(duplicate_reason) run_native_component_accuracy_case( AccuracyEngine, case, @@ -44,6 +53,11 @@ class TestComponentAccuracy2GPU: def test_encoder_accuracy(self, case): if should_skip_component(case, ComponentType.TEXT_ENCODER): pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER)) + duplicate_reason = get_component_duplicate_skip_reason( + case, ComponentType.TEXT_ENCODER + ) + if duplicate_reason: + pytest.skip(duplicate_reason) run_text_encoder_accuracy_case( AccuracyEngine, case, diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index 9dcbff969..681b021c6 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -257,6 +257,7 @@ class DiffusionTestCase: sampling_params: DiffusionSamplingParams run_perf_check: bool = True run_consistency_check: bool = True + run_component_accuracy_check: bool = True run_models_api_check: bool = True run_t2v_input_reference_check: bool = True run_lora_basic_api_check: bool = False @@ -463,6 +464,7 @@ def _make_modelopt_ci_case( sampling_params, run_perf_check=False, run_consistency_check=False, + run_component_accuracy_check=False, ) diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index 9c2de053c..45db47371 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -39,10 +39,6 @@ SGL_TEST_FILES_CONSISTENCY_GT_BASE = SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE SGL_TEST_FILES_CONSISTENCY_GT_BASES = ( SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE, SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE, - # Legacy fallback during migration from sglang-bot/sglang-ci-data - "https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/diffusion-ci/consistency_gt/official_generated", - "https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/diffusion-ci/consistency_gt/sglang_generated", - "https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/diffusion-ci/consistency_gt", ) CONSISTENCY_THRESHOLD_JSON_PATH = ( Path(__file__).resolve().parent / "server" / "consistency_threshold.json" diff --git a/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py b/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py index bce9f31be..f1a5932b3 100644 --- a/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py +++ b/scripts/ci/utils/diffusion/generate_diffusion_dashboard.py @@ -1,6 +1,6 @@ """Generate a Markdown dashboard for diffusion cross-framework comparisons. -Reads current comparison results + historical data from sglang-ci-data repo +Reads current comparison results + historical data from sgl-project/ci-data repo and produces a Markdown report with tables and trend charts saved as PNG files. Usage: @@ -19,16 +19,16 @@ import sys from datetime import datetime, timezone # --------------------------------------------------------------------------- -# History fetching (from sglang-ci-data repo via GitHub API) +# History fetching (from sgl-project/ci-data repo via GitHub API) # --------------------------------------------------------------------------- -CI_DATA_REPO_OWNER = "sglang-bot" -CI_DATA_REPO_NAME = "sglang-ci-data" +CI_DATA_REPO_OWNER = "sgl-project" +CI_DATA_REPO_NAME = "ci-data" CI_DATA_BRANCH = "main" HISTORY_PREFIX = "diffusion-comparisons" MAX_HISTORY_RUNS = 14 -# Base URL for chart images pushed to sglang-ci-data +# Base URL for chart images pushed to sgl-project/ci-data CHARTS_RAW_BASE_URL = ( f"https://raw.githubusercontent.com/{CI_DATA_REPO_OWNER}/{CI_DATA_REPO_NAME}" f"/{CI_DATA_BRANCH}/{HISTORY_PREFIX}/charts" @@ -58,7 +58,7 @@ def _github_get(url: str, token: str) -> dict | list | None: def fetch_history_from_github(token: str) -> list[dict]: - """Fetch recent comparison result JSONs from sglang-ci-data repo.""" + """Fetch recent comparison result JSONs from sgl-project/ci-data repo.""" print("Fetching historical comparison data from GitHub...") url = ( f"https://api.github.com/repos/{CI_DATA_REPO_OWNER}/{CI_DATA_REPO_NAME}" @@ -775,7 +775,7 @@ def main(): parser.add_argument( "--fetch-history", action="store_true", - help="Fetch history from sglang-ci-data GitHub repo", + help="Fetch history from ci-data GitHub repo", ) parser.add_argument( "--step-summary", diff --git a/scripts/ci/utils/diffusion/publish_comparison_results.py b/scripts/ci/utils/diffusion/publish_comparison_results.py index 03a40612c..030f76ef3 100644 --- a/scripts/ci/utils/diffusion/publish_comparison_results.py +++ b/scripts/ci/utils/diffusion/publish_comparison_results.py @@ -1,4 +1,4 @@ -"""Publish diffusion comparison results to sglang-bot/sglang-ci-data repo. +"""Publish diffusion comparison results to sgl-project/ci-data repo. Pushes comparison-results.json, dashboard.md, and chart PNG files to the ci-data repository for historical tracking. Chart PNGs are stored under @@ -49,8 +49,8 @@ else: ) # Repository configuration -REPO_OWNER = "sglang-bot" -REPO_NAME = "sglang-ci-data" +REPO_OWNER = "sgl-project" +REPO_NAME = "ci-data" BRANCH = "main" STORAGE_PREFIX = "diffusion-comparisons" @@ -196,7 +196,7 @@ def publish_comparison( def main(): parser = argparse.ArgumentParser( - description="Publish diffusion comparison results to sglang-ci-data" + description="Publish diffusion comparison results to ci-data" ) parser.add_argument( "--results", diff --git a/scripts/ci/utils/publish_traces.py b/scripts/ci/utils/publish_traces.py index f19fc0cbb..a10f7ef10 100644 --- a/scripts/ci/utils/publish_traces.py +++ b/scripts/ci/utils/publish_traces.py @@ -352,8 +352,8 @@ def publish_traces_from_files(files_to_upload, run_id, run_number): sys.exit(1) # Repository configuration - repo_owner = "sglang-bot" - repo_name = "sglang-ci-data" + repo_owner = "sgl-project" + repo_name = "ci-data" branch = "main" # Verify token permissions before proceeding