From 79f334b1aac3ff4755aa98af4870de41e5f9b145 Mon Sep 17 00:00:00 2001 From: Mick Date: Wed, 1 Jul 2026 19:09:43 +0800 Subject: [PATCH] [diffusion] CI: add 5090 job (#29791) --- .github/workflows/pr-test-multimodal-gen.yml | 63 ++++++ .../test/scripts/gen_perf_baselines.py | 2 +- .../server/consistency_thresholds/5090.json | 10 + .../server/consistency_thresholds/b200.json | 3 + .../h100.json} | 0 .../multimodal_gen/test/server/gpu_cases.py | 77 +++++++ .../test/server/perf_baselines/5090.json | 106 ++++++++++ .../test/server/perf_baselines/b200.json | 79 +++++++ .../h100.json} | 32 --- .../test/server/test_server_1_gpu_5090.py | 20 ++ .../test/server/test_server_common.py | 12 +- .../test/server/testcase_configs.py | 56 ++++- .../sglang/multimodal_gen/test/test_utils.py | 200 +++++++++++++++--- .../test/unit/test_consistency_metrics.py | 47 ++++ .../test/unit/test_turbo_wan_backend.py | 61 ++++++ .../utils/diffusion/diffusion_case_parser.py | 48 +++-- 16 files changed, 731 insertions(+), 85 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/server/consistency_thresholds/5090.json create mode 100644 python/sglang/multimodal_gen/test/server/consistency_thresholds/b200.json rename python/sglang/multimodal_gen/test/server/{consistency_threshold.json => consistency_thresholds/h100.json} (100%) create mode 100644 python/sglang/multimodal_gen/test/server/perf_baselines/5090.json create mode 100644 python/sglang/multimodal_gen/test/server/perf_baselines/b200.json rename python/sglang/multimodal_gen/test/server/{perf_baselines.json => perf_baselines/h100.json} (98%) create mode 100644 python/sglang/multimodal_gen/test/server/test_server_1_gpu_5090.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_turbo_wan_backend.py diff --git a/.github/workflows/pr-test-multimodal-gen.yml b/.github/workflows/pr-test-multimodal-gen.yml index b5cf56347..b831c06e3 100644 --- a/.github/workflows/pr-test-multimodal-gen.yml +++ b/.github/workflows/pr-test-multimodal-gen.yml @@ -144,6 +144,69 @@ jobs: with: artifact-suffix: ${{ matrix.part }} + multimodal-gen-test-1-5090: + if: | + ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) && + inputs.multimodal_gen == 'true' + runs-on: 1-gpu-5090 + timeout-minutes: 180 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.git_ref || github.sha }} + + - uses: ./.github/actions/check-pr-test-health + + - uses: ./.github/actions/check-maintenance + + - name: Download artifacts + if: inputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda* + + - name: Install dependencies + timeout-minutes: 20 + run: | + CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion + + - name: Run 5090 diffusion canary tests + timeout-minutes: 120 + env: + RUNAI_STREAMER_MEMORY_LIMIT: 0 + CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures + run: | + cd python + python3 sglang/multimodal_gen/test/run_suite.py \ + --suite 1-gpu-5090 \ + $CONTINUE_ON_ERROR_FLAG + + - name: Upload execution report + if: always() + uses: actions/upload-artifact@v4 + with: + name: diffusion-5090-report-${{ github.run_attempt }} + path: python/sglang/multimodal_gen/test/execution_report_*.json + retention-days: 7 + + - name: Upload diffusion failure artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: diffusion-failures-5090-${{ github.run_attempt }} + path: diffusion-failures/ + if-no-files-found: ignore + retention-days: 7 + + - uses: ./.github/actions/upload-cuda-coredumps + if: failure() + with: + artifact-suffix: 1-gpu-5090 + multimodal-gen-test-2-gpu: needs: compute-diffusion-partitions if: | diff --git a/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py b/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py index 43e913b6b..1949758b2 100644 --- a/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py +++ b/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py @@ -42,7 +42,7 @@ def _all_cases() -> list[DiffusionTestCase]: def _baseline_path() -> Path: import sglang.multimodal_gen.test.server.testcase_configs as cfg - return Path(cfg.__file__).with_name("perf_baselines.json") + return cfg.get_perf_baseline_path() def _openai_client(port: int) -> OpenAI: diff --git a/python/sglang/multimodal_gen/test/server/consistency_thresholds/5090.json b/python/sglang/multimodal_gen/test/server/consistency_thresholds/5090.json new file mode 100644 index 000000000..9f6f79fa4 --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/consistency_thresholds/5090.json @@ -0,0 +1,10 @@ +{ + "cases": { + "zimage_image_t2i": { + "clip_threshold": 0.92, + "ssim_threshold": 0.86, + "psnr_threshold": 19.5, + "mean_abs_diff_threshold": 8.5 + } + } +} diff --git a/python/sglang/multimodal_gen/test/server/consistency_thresholds/b200.json b/python/sglang/multimodal_gen/test/server/consistency_thresholds/b200.json new file mode 100644 index 000000000..e978a7960 --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/consistency_thresholds/b200.json @@ -0,0 +1,3 @@ +{ + "cases": {} +} diff --git a/python/sglang/multimodal_gen/test/server/consistency_threshold.json b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json similarity index 100% rename from python/sglang/multimodal_gen/test/server/consistency_threshold.json rename to python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index ad256ed60..301a6f4db 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -849,6 +849,80 @@ ONE_GPU_CASES += ONE_GPU_MODELOPT_FP8_CASES TWO_GPU_CASES = _with_default_num_gpus(TWO_GPU_CASES, 2) +ONE_GPU_5090_PERF_CASE_IDS = frozenset( + { + "zimage_image_t2i", + "flux_2_klein_base_image_t2i", + "wan2_1_t2v_1.3b", + } +) +ONE_GPU_5090_SKIP_CONSISTENCY_CASE_IDS = frozenset( + { + "turbo_wan2_1_t2v_1.3b", + } +) + + +def _select_5090_canary_cases(case_ids: tuple[str, ...]) -> list[DiffusionTestCase]: + cases_by_id = {case.id: case for case in ONE_GPU_CASES} + missing = [case_id for case_id in case_ids if case_id not in cases_by_id] + if missing: + raise RuntimeError(f"Unknown 5090 diffusion canary case(s): {missing}") + + return [ + replace( + cases_by_id[case_id], + run_perf_check=case_id in ONE_GPU_5090_PERF_CASE_IDS, + run_consistency_check=( + cases_by_id[case_id].run_consistency_check + and case_id not in ONE_GPU_5090_SKIP_CONSISTENCY_CASE_IDS + ), + ) + for case_id in case_ids + ] + + +def _make_5090_flux_layerwise_cpu_offload_case() -> DiffusionTestCase: + base_case = next(case for case in ONE_GPU_CASES if case.id == "flux_image_t2i") + + return replace( + base_case, + id="flux_image_t2i_layerwise_cpu_offload_5090", + server_args=replace( + base_case.server_args, + dit_layerwise_offload=True, + dit_offload_prefetch_size=5, + text_encoder_cpu_offload=True, + extras=[ + *base_case.server_args.extras, + "--dit-cpu-offload", + "--pin-cpu-memory", + ], + ), + sampling_params=replace( + T2I_sampling_params, + output_size="512x512", + extras={"num_inference_steps": 4, "seed": 0}, + ), + run_perf_check=False, + run_consistency_check=False, + run_component_accuracy_check=False, + run_models_api_check=False, + run_t2v_input_reference_check=False, + ) + + +ONE_GPU_5090_CASES = _select_5090_canary_cases( + ( + "zimage_image_t2i", + "flux_2_klein_base_image_t2i", + "wan2_1_t2v_1.3b", + "turbo_wan2_1_t2v_1.3b", + ) +) +ONE_GPU_5090_CASES.append(_make_5090_flux_layerwise_cpu_offload_case()) + + def _discover_unit_tests() -> list[str]: unit_dir = Path(__file__).resolve().parent.parent / "unit" if not unit_dir.is_dir(): @@ -879,6 +953,9 @@ PARAMETRIZED_CASE_GROUPS = { "1-gpu": [ ("test_server_1_gpu.py", ONE_GPU_CASES), ], + "1-gpu-5090": [ + ("test_server_1_gpu_5090.py", ONE_GPU_5090_CASES), + ], "2-gpu": [ ("test_server_2_gpu.py", TWO_GPU_CASES), ], diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/5090.json b/python/sglang/multimodal_gen/test/server/perf_baselines/5090.json new file mode 100644 index 000000000..4205f901c --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/5090.json @@ -0,0 +1,106 @@ +{ + "metadata": { + "model": "Diffusion Server", + "hardware": "CI RTX 5090 pool", + "description": "Reference numbers captured from real 5090 CI runner history (PR 29791 run 28492141274, job 84450974283).", + "last_updated": "2026-07-01" + }, + "tolerances": { + "long_term": { + "e2e": 0.15, + "denoise_stage": 0.1, + "non_denoise_stage": 0.5, + "denoise_step": 0.25, + "denoise_agg": 0.15 + }, + "pr_test": { + "e2e": 0.25, + "denoise_stage": 0.25, + "non_denoise_stage": 0.8, + "denoise_step": 0.3, + "denoise_agg": 0.2 + } + }, + "improvement_reporting": { + "threshold": 0.2 + }, + "sampling": { + "step_fractions": [ + 0.0, + 0.2, + 0.4, + 0.6, + 0.8, + 1.0 + ] + }, + "scenarios": { + "flux_2_klein_base_image_t2i": { + "stages_ms": { + "DecodingStage": 18.19, + "DenoisingStage": 18213.48, + "ImageVAEEncodingStage": 0.01, + "InputValidationStage": 0.08, + "LatentPreparationStage": 4.87, + "TextEncodingStage": 104.46, + "TimestepPreparationStage": 142.18 + }, + "denoise_step_ms": { + "0": 219.58, + "10": 359.79, + "20": 361.55, + "29": 361.99, + "39": 362.57, + "49": 362.89 + }, + "expected_e2e_ms": 18641.09, + "expected_avg_denoise_ms": 357.93, + "expected_median_denoise_ms": 361.8, + "estimated_full_test_time_s": 94.0 + }, + "wan2_1_t2v_1.3b": { + "stages_ms": { + "DecodingStage": 1202.43, + "DenoisingStage": 21452.59, + "InputValidationStage": 0.09, + "LatentPreparationStage": 0.23, + "TextEncodingStage": 1210.22, + "TimestepPreparationStage": 3.91 + }, + "denoise_step_ms": { + "0": 480.39, + "10": 426.11, + "20": 427.32, + "29": 428.09, + "39": 428.04, + "49": 422.31 + }, + "expected_e2e_ms": 23877.73, + "expected_avg_denoise_ms": 428.74, + "expected_median_denoise_ms": 427.94, + "estimated_full_test_time_s": 160.9 + }, + "zimage_image_t2i": { + "stages_ms": { + "DecodingStage": 7.11, + "DenoisingStage": 2229.48, + "InputValidationStage": 0.04, + "LatentPreparationStage": 0.17, + "TextEncodingStage": 252.73, + "TimestepPreparationStage": 38.57 + }, + "denoise_step_ms": { + "0": 39.34, + "2": 277.35, + "3": 267.94, + "5": 279.01, + "6": 273.01, + "8": 275.12 + }, + "expected_e2e_ms": 2533.95, + "expected_avg_denoise_ms": 246.97, + "expected_median_denoise_ms": 273.01, + "estimated_full_test_time_s": 329.8 + } + } +} diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json b/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json new file mode 100644 index 000000000..f99923e55 --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/b200.json @@ -0,0 +1,79 @@ +{ + "metadata": { + "model": "Diffusion Server", + "hardware": "CI B200 pool", + "description": "Reference estimates for B200-only diffusion cases, split out from the shared diffusion baseline file.", + "last_updated": "2026-07-01" + }, + "tolerances": { + "long_term": { + "e2e": 0.15, + "denoise_stage": 0.1, + "non_denoise_stage": 0.5, + "denoise_step": 0.25, + "denoise_agg": 0.15 + }, + "pr_test": { + "e2e": 0.25, + "denoise_stage": 0.25, + "non_denoise_stage": 0.8, + "denoise_step": 0.3, + "denoise_agg": 0.2 + } + }, + "improvement_reporting": { + "threshold": 0.2 + }, + "sampling": { + "step_fractions": [ + 0.0, + 0.2, + 0.4, + 0.6, + 0.8, + 1.0 + ] + }, + "scenarios": { + "flux1_modelopt_nvfp4_t2i": { + "stages_ms": {}, + "denoise_step_ms": {}, + "expected_e2e_ms": 0.0, + "expected_avg_denoise_ms": 0.0, + "expected_median_denoise_ms": 0.0, + "estimated_full_test_time_s": 71.2 + }, + "flux2_modelopt_nvfp4_t2i": { + "stages_ms": {}, + "denoise_step_ms": {}, + "expected_e2e_ms": 0.0, + "expected_avg_denoise_ms": 0.0, + "expected_median_denoise_ms": 0.0, + "estimated_full_test_time_s": 592.3 + }, + "qwen_image_2512_modelopt_nvfp4_t2i": { + "stages_ms": {}, + "denoise_step_ms": {}, + "expected_e2e_ms": 0.0, + "expected_avg_denoise_ms": 0.0, + "expected_median_denoise_ms": 0.0, + "estimated_full_test_time_s": 120.0 + }, + "wan22_modelopt_nvfp4_t2v": { + "stages_ms": {}, + "denoise_step_ms": {}, + "expected_e2e_ms": 0.0, + "expected_avg_denoise_ms": 0.0, + "expected_median_denoise_ms": 0.0, + "estimated_full_test_time_s": 181.8 + }, + "ideogram4_nvfp4_t2i": { + "stages_ms": {}, + "denoise_step_ms": {}, + "expected_e2e_ms": 0.0, + "expected_avg_denoise_ms": 0.0, + "expected_median_denoise_ms": 0.0, + "estimated_full_test_time_s": 300.0 + } + } +} diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json similarity index 98% rename from python/sglang/multimodal_gen/test/server/perf_baselines.json rename to python/sglang/multimodal_gen/test/server/perf_baselines/h100.json index 8cb8d7964..ebc8e4824 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json @@ -2829,38 +2829,6 @@ "expected_avg_denoise_ms": 0.0, "expected_median_denoise_ms": 0.0, "estimated_full_test_time_s": 73.7 - }, - "flux1_modelopt_nvfp4_t2i": { - "stages_ms": {}, - "denoise_step_ms": {}, - "expected_e2e_ms": 0.0, - "expected_avg_denoise_ms": 0.0, - "expected_median_denoise_ms": 0.0, - "estimated_full_test_time_s": 71.2 - }, - "flux2_modelopt_nvfp4_t2i": { - "stages_ms": {}, - "denoise_step_ms": {}, - "expected_e2e_ms": 0.0, - "expected_avg_denoise_ms": 0.0, - "expected_median_denoise_ms": 0.0, - "estimated_full_test_time_s": 592.3 - }, - "qwen_image_2512_modelopt_nvfp4_t2i": { - "stages_ms": {}, - "denoise_step_ms": {}, - "expected_e2e_ms": 0.0, - "expected_avg_denoise_ms": 0.0, - "expected_median_denoise_ms": 0.0, - "estimated_full_test_time_s": 120.0 - }, - "wan22_modelopt_nvfp4_t2v": { - "stages_ms": {}, - "denoise_step_ms": {}, - "expected_e2e_ms": 0.0, - "expected_avg_denoise_ms": 0.0, - "expected_median_denoise_ms": 0.0, - "estimated_full_test_time_s": 181.8 } } } diff --git a/python/sglang/multimodal_gen/test/server/test_server_1_gpu_5090.py b/python/sglang/multimodal_gen/test/server/test_server_1_gpu_5090.py new file mode 100644 index 000000000..6de03df51 --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/test_server_1_gpu_5090.py @@ -0,0 +1,20 @@ +""" +Config-driven diffusion canary tests for the 1-GPU 5090 PR runner. +""" + +from __future__ import annotations + +from sglang.multimodal_gen.test.server.common.case_fixtures import ( + diffusion_case_fixture, +) +from sglang.multimodal_gen.test.server.gpu_cases import ONE_GPU_5090_CASES +from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 + DiffusionServerBase, + diffusion_server, +) + + +class TestDiffusionServerOneGpu5090(DiffusionServerBase): + """Canary tests for lightweight 1-GPU diffusion cases on 5090.""" + + case = diffusion_case_fixture(ONE_GPU_5090_CASES) diff --git a/python/sglang/multimodal_gen/test/server/test_server_common.py b/python/sglang/multimodal_gen/test/server/test_server_common.py index 537ab13e9..0542588e1 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_common.py +++ b/python/sglang/multimodal_gen/test/server/test_server_common.py @@ -41,6 +41,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import ( PerformanceSummary, ScenarioConfig, get_model_task_type_for_server_args, + get_perf_baseline_path, ) from sglang.multimodal_gen.test.test_utils import ( SGL_TEST_FILES_CI_DATA_REVISION, @@ -50,6 +51,7 @@ from sglang.multimodal_gen.test.test_utils import ( extract_key_frames_from_video, get_consistency_gt_candidates, get_consistency_gt_remote_files, + get_consistency_threshold_path, get_consistency_thresholds, get_dynamic_server_port, gt_exists, @@ -232,7 +234,7 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext: logger.error( f'\n{"=" * 60}\n' f'Add "estimated_full_test_time_s" to scenario "{case.id}":\n\n' - f"File: python/sglang/multimodal_gen/test/server/perf_baselines.json\n\n" + f"File: {get_perf_baseline_path()}\n\n" f' "{case.id}": {{\n' f" ...\n" f' "estimated_full_test_time_s": {_measured_full_time:.1f}\n' @@ -441,7 +443,7 @@ class DiffusionServerBase: self._dump_baseline_for_testcase(case, summary, missing_scenario) if missing_scenario: pytest.fail( - f"Testcase '{case.id}' not found in perf_baselines.json" + f"Testcase '{case.id}' not found in {get_perf_baseline_path()}" ) return @@ -552,7 +554,7 @@ class DiffusionServerBase: ) action = "add" if missing_scenario else "update" output = f""" -{action} this baseline in the "scenarios" section of perf_baselines.json: +{action} this baseline in the "scenarios" section of {get_perf_baseline_path()}: "{case.id}": {json.dumps(baseline, indent=4)} @@ -607,10 +609,10 @@ Add the expected file(s) to sgl-project/ci-data in diffusion-ci/consistency_gt/s For this case, expected file(s): {names} -Repository: https://github.com/sgl-project/ci-data (path: diffusion-ci/consistency_gt/sglang_generated/) +Repository: https://github.com/sgl-project/ci-data (path: diffusion-ci/consistency_gt/sglang_generated/, with optional platform subdirectories such as 5090/) Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION} -(Optional) Per-case override in consistency_threshold.json: +(Optional) Per-case override in {get_consistency_threshold_path()}: "cases": {{ "{case.id}": {{ "clip_threshold": 0.92, diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index fad5e4198..c5b0ae132 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -12,7 +12,7 @@ pytest python/sglang/multimodal_gen/test/server/test_server_1_gpu.py -k qwen_ima To add a new testcase: 1. add your testcase with case-id: `my_new_test_case_id` to `ONE_GPU_CASES`, `ONE_GPU_MODELOPT_FP8_CASES`, `ONE_GPU_B200_CASES`, or `TWO_GPU_CASES` 2. run `SGLANG_GEN_BASELINE=1 pytest -s python/sglang/multimodal_gen/test/server/ -k my_new_test_case_id` -3. insert or override the corresponding scenario in `scenarios` section of perf_baselines.json with the output baseline of step-2 +3. insert or override the corresponding scenario in the platform JSON under `perf_baselines/` """ @@ -33,6 +33,7 @@ from sglang.multimodal_gen.registry import ( get_model_info, get_pipeline_config_classes, ) +from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord @@ -657,6 +658,57 @@ MODELOPT_WAN22_NVFP4_MODEL = "nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4" MODELOPT_NVFP4_B200_ENV_VARS = {} MODELOPT_WAN22_NVFP4_B200_ENV_VARS = {} +PERF_BASELINE_PLATFORM_ENV = "SGLANG_DIFFUSION_PERF_BASELINE_PLATFORM" +PERF_BASELINE_DIR = Path(__file__).with_name("perf_baselines") +PERF_BASELINE_FILE_BY_PLATFORM = { + "h100": "h100.json", + "b200": "b200.json", + "5090": "5090.json", +} +PERF_BASELINE_PLATFORM_ALIASES = { + "sm90": "h100", + "hopper": "h100", + "h100": "h100", + "sm100": "b200", + "blackwell": "b200", + "b200": "b200", + "sm120": "5090", + "rtx5090": "5090", + "5090": "5090", +} + + +def _normalize_perf_baseline_platform(platform: str) -> str: + normalized = platform.strip().lower().replace("_", "-") + normalized = normalized.replace("-", "") + if normalized not in PERF_BASELINE_PLATFORM_ALIASES: + valid = ", ".join(sorted(PERF_BASELINE_FILE_BY_PLATFORM)) + raise ValueError( + f"Invalid diffusion perf baseline platform {platform!r}. " + f"Expected one of: {valid}" + ) + return PERF_BASELINE_PLATFORM_ALIASES[normalized] + + +def get_perf_baseline_platform() -> str: + override = os.getenv(PERF_BASELINE_PLATFORM_ENV) + if override: + return _normalize_perf_baseline_platform(override) + if current_platform.is_sm120(): + return "5090" + if current_platform.is_blackwell(): + return "b200" + return "h100" + + +def get_perf_baseline_path(platform: str | None = None) -> Path: + baseline_platform = ( + _normalize_perf_baseline_platform(platform) + if platform is not None + else get_perf_baseline_platform() + ) + return PERF_BASELINE_DIR / PERF_BASELINE_FILE_BY_PLATFORM[baseline_platform] + def _make_modelopt_ci_case( case_id: str, @@ -694,7 +746,7 @@ def _with_default_num_gpus( # Load global configuration BASELINE_CONFIG = ( - BaselineConfig.load(Path(__file__).with_name("perf_baselines.json")) + BaselineConfig.load(get_perf_baseline_path()) .update(Path(__file__).parent / "ascend" / "perf_baselines_npu.json") .update(Path(__file__).parent / "musa" / "perf_baselines_musa.json") ) diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index 989f229ef..ade24cdd5 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -34,7 +34,7 @@ if TYPE_CHECKING: logger = init_logger(__name__) -SGL_TEST_FILES_CI_DATA_REVISION = "4a271ef34602043f19d253f0d30a5f653fe11325" +SGL_TEST_FILES_CI_DATA_REVISION = "702d939e23f17b42183329dace60f221d2587056" if current_platform.is_npu(): SGL_TEST_FILES_CI_DATA_REVISION = "670d66a8a290b62c0c3c077b3e9b0f4a4d9a44e7" @@ -79,9 +79,26 @@ SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_CASES = frozenset( } ) -CONSISTENCY_THRESHOLD_JSON_PATH = ( - Path(__file__).resolve().parent / "server" / "consistency_threshold.json" +CONSISTENCY_PLATFORM_ENV = "SGLANG_DIFFUSION_CONSISTENCY_PLATFORM" +CONSISTENCY_THRESHOLD_DIR = ( + Path(__file__).resolve().parent / "server" / "consistency_thresholds" ) +CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM = { + "h100": "h100.json", + "b200": "b200.json", + "5090": "5090.json", +} +CONSISTENCY_PLATFORM_ALIASES = { + "sm90": "h100", + "hopper": "h100", + "h100": "h100", + "sm100": "b200", + "blackwell": "b200", + "b200": "b200", + "sm120": "5090", + "rtx5090": "5090", + "5090": "5090", +} CLIP_MODEL_NAME = "openai/clip-vit-large-patch14" DEFAULT_CLIP_THRESHOLD_IMAGE = 0.92 DEFAULT_CLIP_THRESHOLD_VIDEO = 0.90 @@ -694,14 +711,74 @@ def validate_video_file( ), f"Video height mismatch: expected {expected_height}, got {actual_height}" -def _load_threshold_json() -> dict[str, Any]: - """Load consistency_threshold.json; returns {} if missing.""" - if not CONSISTENCY_THRESHOLD_JSON_PATH.exists(): +def _normalize_consistency_platform(platform: str) -> str: + normalized = platform.strip().lower().replace("_", "-") + normalized = normalized.replace("-", "") + if normalized not in CONSISTENCY_PLATFORM_ALIASES: + valid = ", ".join(sorted(CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM)) + raise ValueError( + f"Invalid diffusion consistency platform {platform!r}. " + f"Expected one of: {valid}" + ) + return CONSISTENCY_PLATFORM_ALIASES[normalized] + + +def get_consistency_platform() -> str: + override = os.getenv(CONSISTENCY_PLATFORM_ENV) + if override: + return _normalize_consistency_platform(override) + if current_platform.is_sm120(): + return "5090" + if current_platform.is_blackwell(): + return "b200" + return "h100" + + +def get_consistency_threshold_path(platform: str | None = None) -> Path: + threshold_platform = ( + _normalize_consistency_platform(platform) + if platform is not None + else get_consistency_platform() + ) + return ( + CONSISTENCY_THRESHOLD_DIR + / CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM[threshold_platform] + ) + + +def _load_threshold_file(path: Path) -> dict[str, Any]: + if not path.exists(): return {} - with CONSISTENCY_THRESHOLD_JSON_PATH.open("r", encoding="utf-8") as f: + with path.open("r", encoding="utf-8") as f: return json.load(f) +def _merge_threshold_metadata( + base: dict[str, Any], override: dict[str, Any] +) -> dict[str, Any]: + merged = dict(base) + if "cases" in base or "cases" in override: + merged["cases"] = { + **base.get("cases", {}), + **override.get("cases", {}), + } + for key, value in override.items(): + if key != "cases": + merged[key] = value + return merged + + +def _load_threshold_json() -> dict[str, Any]: + metadata = _load_threshold_file(get_consistency_threshold_path("h100")) + platform = get_consistency_platform() + if platform == "h100": + return metadata + return _merge_threshold_metadata( + metadata, + _load_threshold_file(get_consistency_threshold_path(platform)), + ) + + @dataclass class ConsistencyThresholds: clip_threshold: float @@ -958,10 +1035,9 @@ def _consistency_gt_filenames( return [f"{case_id}_{n}gpu.{ext}"] -def get_consistency_gt_candidates( +def _base_consistency_gt_candidates( case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None ) -> list[str]: - """Return candidate GT filenames for local consistency data.""" n = num_gpus if is_video: return [ @@ -975,6 +1051,31 @@ def get_consistency_gt_candidates( return [f"{base}.{e}" for e in exts] +def get_consistency_gt_candidate_sets( + case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None +) -> list[list[str]]: + candidates = _base_consistency_gt_candidates( + case_id, num_gpus, is_video, output_format + ) + platform = get_consistency_platform() + if platform == "h100": + return [candidates] + return [[f"{platform}/{candidate}" for candidate in candidates], candidates] + + +def get_consistency_gt_candidates( + case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None +) -> list[str]: + """Return candidate GT filenames for local consistency data.""" + return [ + candidate + for candidate_set in get_consistency_gt_candidate_sets( + case_id, num_gpus, is_video, output_format + ) + for candidate in candidate_set + ] + + def get_consistency_gt_remote_files( case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None ) -> list[tuple[str, str]]: @@ -1003,6 +1104,21 @@ def _remote_consistency_gt_candidates( return [(filename, f"{base_url}/{filename}") for filename in filenames] +def _remote_consistency_gt_candidate_sets( + base_url: str, + case_id: str, + num_gpus: int, + is_video: bool, + output_format: str | None = None, +) -> list[list[tuple[str, str]]]: + return [ + [(filename, f"{base_url}/{filename}") for filename in filenames] + for filenames in get_consistency_gt_candidate_sets( + case_id, num_gpus, is_video, output_format + ) + ] + + def _is_ascend_consistency_case(case_id: str) -> bool: return "npu" in case_id @@ -1086,14 +1202,15 @@ def _find_remote_consistency_gt_files( # Avoid accidentally comparing non-comparable CI cases against official GT. bases = (SGL_TEST_FILES_CONSISTENCY_GT_BASE,) for base_url in bases: - candidates = _remote_consistency_gt_candidates( + candidate_sets = _remote_consistency_gt_candidate_sets( base_url, case_id, num_gpus, is_video, output_format ) - if is_video: - exists = [_remote_file_exists(url) for _, url in candidates] - if all(status is not False for status in exists): - return candidates - else: + for candidates in candidate_sets: + if is_video: + exists = [_remote_file_exists(url) for _, url in candidates] + if all(status is not False for status in exists): + return candidates + continue uncertain_candidate = None for filename, url in candidates: exists = _remote_file_exists(url) @@ -1122,7 +1239,8 @@ def _get_consistency_gt_cache_key( ) -> str: gt_dir = _get_consistency_gt_dir() source = str(gt_dir) if gt_dir is not None else "remote" - return f"{case_id}:{num_gpus}:{is_video}:{output_format or ''}:{source}" + platform = get_consistency_platform() + return f"{platform}:{case_id}:{num_gpus}:{is_video}:{output_format or ''}:{source}" def load_consistency_gt( @@ -1139,29 +1257,43 @@ def load_consistency_gt( if cached is not None: return cached - filenames = _consistency_gt_filenames(case_id, num_gpus, is_video, output_format) images: list[np.ndarray] = [] gt_dir = _get_consistency_gt_dir() if gt_dir is not None: - candidates = get_consistency_gt_candidates( + candidate_sets = get_consistency_gt_candidate_sets( case_id, num_gpus, is_video, output_format ) if is_video: - for fn in candidates: - path = gt_dir / fn - if not path.exists(): - raise FileNotFoundError(f"GT image not found: {path}") - arr = np.array(Image.open(path).convert("RGB")) - images.append(arr) + selected = None + for candidates in candidate_sets: + if all((gt_dir / fn).exists() for fn in candidates): + selected = candidates + break + if selected is None: + tried = ", ".join( + candidate + for candidates in candidate_sets + for candidate in candidates + ) + raise FileNotFoundError( + f"GT images not found in {gt_dir}. Tried: {tried}" + ) + for fn in selected: + images.append(np.array(Image.open(gt_dir / fn).convert("RGB"))) else: path = None - for fn in candidates: + for fn in get_consistency_gt_candidates( + case_id, num_gpus, is_video, output_format + ): candidate = gt_dir / fn if candidate.exists(): path = candidate break if path is None: + candidates = get_consistency_gt_candidates( + case_id, num_gpus, is_video, output_format + ) raise FileNotFoundError( f"GT image not found in {gt_dir}. Tried: {', '.join(candidates)}" ) @@ -1172,8 +1304,11 @@ def load_consistency_gt( case_id, num_gpus, is_video, output_format ) if not remote_files: + candidates = get_consistency_gt_candidates( + case_id, num_gpus, is_video, output_format + ) raise FileNotFoundError( - f"GT image not found for {case_id}. Tried: {', '.join(filenames)}" + f"GT image not found for {case_id}. Tried: {', '.join(candidates)}" ) for _, url in remote_files: images.append(_load_remote_gt_image(url)) @@ -1210,12 +1345,19 @@ def gt_exists( """Check whether GT image(s) exist.""" gt_dir = _get_consistency_gt_dir() if gt_dir is not None: - candidates = get_consistency_gt_candidates( + candidate_sets = get_consistency_gt_candidate_sets( case_id, num_gpus, is_video, output_format ) if is_video: - return all((gt_dir / c).exists() for c in candidates) - return any((gt_dir / c).exists() for c in candidates) + return any( + all((gt_dir / candidate).exists() for candidate in candidate_set) + for candidate_set in candidate_sets + ) + return any( + (gt_dir / candidate).exists() + for candidate_set in candidate_sets + for candidate in candidate_set + ) cache_key = _get_consistency_gt_cache_key( case_id, num_gpus, is_video, output_format diff --git a/python/sglang/multimodal_gen/test/unit/test_consistency_metrics.py b/python/sglang/multimodal_gen/test/unit/test_consistency_metrics.py index 44ba0a89f..15992f6ff 100644 --- a/python/sglang/multimodal_gen/test/unit/test_consistency_metrics.py +++ b/python/sglang/multimodal_gen/test/unit/test_consistency_metrics.py @@ -54,6 +54,53 @@ def test_remote_video_gt_candidates_survive_inconclusive_probe(monkeypatch): ] +def test_platform_gt_candidates_prefer_platform_then_default(monkeypatch): + monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "5090") + + assert test_utils.get_consistency_gt_candidates( + "unit_image", + 1, + is_video=False, + output_format="png", + ) == [ + "5090/unit_image_1gpu.png", + "5090/unit_image_1gpu.jpg", + "5090/unit_image_1gpu.webp", + "unit_image_1gpu.png", + "unit_image_1gpu.jpg", + "unit_image_1gpu.webp", + ] + + +def test_threshold_metadata_merges_platform_override(): + metadata = test_utils._merge_threshold_metadata( + { + "cases": { + "case_a": { + "clip_threshold": 0.9, + "ssim_threshold": 0.9, + "psnr_threshold": 20.0, + "mean_abs_diff_threshold": 10.0, + } + }, + "default_clip_threshold_image": 0.92, + }, + { + "cases": { + "case_a": { + "clip_threshold": 0.8, + "ssim_threshold": 0.7, + "psnr_threshold": 12.0, + "mean_abs_diff_threshold": 20.0, + } + } + }, + ) + + assert metadata["default_clip_threshold_image"] == 0.92 + assert metadata["cases"]["case_a"]["psnr_threshold"] == 12.0 + + def test_pixel_metrics_identical_image(): image = _solid_image(128) diff --git a/python/sglang/multimodal_gen/test/unit/test_turbo_wan_backend.py b/python/sglang/multimodal_gen/test/unit/test_turbo_wan_backend.py new file mode 100644 index 000000000..214f2d549 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_turbo_wan_backend.py @@ -0,0 +1,61 @@ +import unittest + +from sglang.multimodal_gen.runtime.layers.attention.turbo_layer import ( + _resolve_turbo_wan_sparse_backend, +) +from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum + + +class TestTurboWanBackendSelection(unittest.TestCase): + def test_non_sparse_requested_backend_falls_back_to_attention_type(self): + selected, warning = _resolve_turbo_wan_sparse_backend( + attention_type="sla", + requested_attention_backend="fa", + ) + + self.assertEqual(selected, AttentionBackendEnum.SLA_ATTN) + self.assertIsNotNone(warning) + self.assertIn("TurboWan only supports", warning) + self.assertIn("attention_backend='fa'", warning) + + def test_sagesla_attention_type_prefers_sage_sparse_backend(self): + selected, warning = _resolve_turbo_wan_sparse_backend( + attention_type="sagesla", + requested_attention_backend="torch_sdpa", + ) + + self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN) + self.assertIsNotNone(warning) + + def test_requested_sparse_backend_is_honored(self): + selected, warning = _resolve_turbo_wan_sparse_backend( + attention_type="sla", + requested_attention_backend="sage_sla_attn", + ) + + self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN) + self.assertIsNone(warning) + + def test_supported_backend_filter_is_respected(self): + selected, warning = _resolve_turbo_wan_sparse_backend( + attention_type="sla", + requested_attention_backend=None, + supported_attention_backends={AttentionBackendEnum.SAGE_SLA_ATTN}, + ) + + self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN) + self.assertIsNone(warning) + + def test_empty_supported_backend_intersection_keeps_turbowan_choices(self): + selected, warning = _resolve_turbo_wan_sparse_backend( + attention_type="sla", + requested_attention_backend=None, + supported_attention_backends={AttentionBackendEnum.FA}, + ) + + self.assertEqual(selected, AttentionBackendEnum.SLA_ATTN) + self.assertIsNone(warning) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/utils/diffusion/diffusion_case_parser.py b/scripts/ci/utils/diffusion/diffusion_case_parser.py index 0aba7780c..81af8756c 100755 --- a/scripts/ci/utils/diffusion/diffusion_case_parser.py +++ b/scripts/ci/utils/diffusion/diffusion_case_parser.py @@ -42,7 +42,8 @@ DEFAULT_EST_TIME_SECONDS = 300.0 STARTUP_OVERHEAD_SECONDS = 120.0 # Paths relative to repository root -BASELINE_REL_PATH = "python/sglang/multimodal_gen/test/server/perf_baselines.json" +BASELINE_REL_PATH = "python/sglang/multimodal_gen/test/server/perf_baselines" +BASELINE_PLATFORM_ORDER = ("h100", "b200", "5090") RUN_SUITE_REL_PATH = "python/sglang/multimodal_gen/test/run_suite.py" USE_NPU_CONFIGS = os.getenv("USE_NPU_CONFIGS", "0").lower() in ("1", "true") @@ -345,28 +346,43 @@ class RunSuiteVisitor(ast.NodeVisitor): return result +def _iter_baseline_paths(baseline_path: Path) -> List[Path]: + if baseline_path.is_file(): + return [baseline_path] + if not baseline_path.is_dir(): + return [] + + ordered_paths = [ + baseline_path / f"{platform}.json" for platform in BASELINE_PLATFORM_ORDER + ] + ordered_paths.extend( + path + for path in sorted(baseline_path.glob("*.json")) + if path not in ordered_paths + ) + return [path for path in ordered_paths if path.exists()] + + def load_baselines(baseline_path: Path) -> Dict[str, float]: """ - Load performance baselines from JSON file. + Load performance baselines from a JSON file or platform baseline directory. Returns: Dictionary mapping case_id to estimated time in seconds. """ - if not baseline_path.exists(): - return {} - - with open(baseline_path, "r", encoding="utf-8") as f: - data = json.load(f) - baselines = {} - scenarios = data.get("scenarios", {}) + for path in _iter_baseline_paths(baseline_path): + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) - for case_id, scenario in scenarios.items(): - if scenario.get("estimated_full_test_time_s") is not None: - baselines[case_id] = scenario["estimated_full_test_time_s"] - else: - expected_e2e_ms = scenario.get("expected_e2e_ms", 0) - baselines[case_id] = expected_e2e_ms / 1000.0 + STARTUP_OVERHEAD_SECONDS + scenarios = data.get("scenarios", {}) + for case_id, scenario in scenarios.items(): + if scenario.get("estimated_full_test_time_s") is not None: + est_time = scenario["estimated_full_test_time_s"] + else: + expected_e2e_ms = scenario.get("expected_e2e_ms", 0) + est_time = expected_e2e_ms / 1000.0 + STARTUP_OVERHEAD_SECONDS + baselines.setdefault(case_id, est_time) return baselines @@ -443,7 +459,7 @@ def collect_diffusion_suites( Args: case_config_path: Path to case config (resolved from run_suite.py) run_suite_path: Path to run_suite.py - baseline_path: Path to perf_baselines.json + baseline_path: Path to perf_baselines/ or a single baseline JSON file Returns: Dictionary mapping suite name to DiffusionSuiteInfo.