[diffusion] CI: add 5090 job (#29791)
This commit is contained in:
@@ -144,6 +144,69 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
artifact-suffix: ${{ matrix.part }}
|
artifact-suffix: ${{ matrix.part }}
|
||||||
|
|
||||||
|
multimodal-gen-test-1-5090:
|
||||||
|
if: |
|
||||||
|
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
|
||||||
|
inputs.multimodal_gen == 'true'
|
||||||
|
runs-on: 1-gpu-5090
|
||||||
|
timeout-minutes: 180
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.git_ref || github.sha }}
|
||||||
|
|
||||||
|
- uses: ./.github/actions/check-pr-test-health
|
||||||
|
|
||||||
|
- uses: ./.github/actions/check-maintenance
|
||||||
|
|
||||||
|
- name: Download artifacts
|
||||||
|
if: inputs.sgl_kernel == 'true'
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
path: sgl-kernel/dist/
|
||||||
|
merge-multiple: true
|
||||||
|
pattern: wheel-python3.10-cuda*
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
timeout-minutes: 20
|
||||||
|
run: |
|
||||||
|
CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
|
||||||
|
|
||||||
|
- name: Run 5090 diffusion canary tests
|
||||||
|
timeout-minutes: 120
|
||||||
|
env:
|
||||||
|
RUNAI_STREAMER_MEMORY_LIMIT: 0
|
||||||
|
CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||||
|
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
|
||||||
|
run: |
|
||||||
|
cd python
|
||||||
|
python3 sglang/multimodal_gen/test/run_suite.py \
|
||||||
|
--suite 1-gpu-5090 \
|
||||||
|
$CONTINUE_ON_ERROR_FLAG
|
||||||
|
|
||||||
|
- name: Upload execution report
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: diffusion-5090-report-${{ github.run_attempt }}
|
||||||
|
path: python/sglang/multimodal_gen/test/execution_report_*.json
|
||||||
|
retention-days: 7
|
||||||
|
|
||||||
|
- name: Upload diffusion failure artifacts
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: diffusion-failures-5090-${{ github.run_attempt }}
|
||||||
|
path: diffusion-failures/
|
||||||
|
if-no-files-found: ignore
|
||||||
|
retention-days: 7
|
||||||
|
|
||||||
|
- uses: ./.github/actions/upload-cuda-coredumps
|
||||||
|
if: failure()
|
||||||
|
with:
|
||||||
|
artifact-suffix: 1-gpu-5090
|
||||||
|
|
||||||
multimodal-gen-test-2-gpu:
|
multimodal-gen-test-2-gpu:
|
||||||
needs: compute-diffusion-partitions
|
needs: compute-diffusion-partitions
|
||||||
if: |
|
if: |
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ def _all_cases() -> list[DiffusionTestCase]:
|
|||||||
def _baseline_path() -> Path:
|
def _baseline_path() -> Path:
|
||||||
import sglang.multimodal_gen.test.server.testcase_configs as cfg
|
import sglang.multimodal_gen.test.server.testcase_configs as cfg
|
||||||
|
|
||||||
return Path(cfg.__file__).with_name("perf_baselines.json")
|
return cfg.get_perf_baseline_path()
|
||||||
|
|
||||||
|
|
||||||
def _openai_client(port: int) -> OpenAI:
|
def _openai_client(port: int) -> OpenAI:
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"cases": {
|
||||||
|
"zimage_image_t2i": {
|
||||||
|
"clip_threshold": 0.92,
|
||||||
|
"ssim_threshold": 0.86,
|
||||||
|
"psnr_threshold": 19.5,
|
||||||
|
"mean_abs_diff_threshold": 8.5
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"cases": {}
|
||||||
|
}
|
||||||
@@ -849,6 +849,80 @@ ONE_GPU_CASES += ONE_GPU_MODELOPT_FP8_CASES
|
|||||||
TWO_GPU_CASES = _with_default_num_gpus(TWO_GPU_CASES, 2)
|
TWO_GPU_CASES = _with_default_num_gpus(TWO_GPU_CASES, 2)
|
||||||
|
|
||||||
|
|
||||||
|
ONE_GPU_5090_PERF_CASE_IDS = frozenset(
|
||||||
|
{
|
||||||
|
"zimage_image_t2i",
|
||||||
|
"flux_2_klein_base_image_t2i",
|
||||||
|
"wan2_1_t2v_1.3b",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
ONE_GPU_5090_SKIP_CONSISTENCY_CASE_IDS = frozenset(
|
||||||
|
{
|
||||||
|
"turbo_wan2_1_t2v_1.3b",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _select_5090_canary_cases(case_ids: tuple[str, ...]) -> list[DiffusionTestCase]:
|
||||||
|
cases_by_id = {case.id: case for case in ONE_GPU_CASES}
|
||||||
|
missing = [case_id for case_id in case_ids if case_id not in cases_by_id]
|
||||||
|
if missing:
|
||||||
|
raise RuntimeError(f"Unknown 5090 diffusion canary case(s): {missing}")
|
||||||
|
|
||||||
|
return [
|
||||||
|
replace(
|
||||||
|
cases_by_id[case_id],
|
||||||
|
run_perf_check=case_id in ONE_GPU_5090_PERF_CASE_IDS,
|
||||||
|
run_consistency_check=(
|
||||||
|
cases_by_id[case_id].run_consistency_check
|
||||||
|
and case_id not in ONE_GPU_5090_SKIP_CONSISTENCY_CASE_IDS
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for case_id in case_ids
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _make_5090_flux_layerwise_cpu_offload_case() -> DiffusionTestCase:
|
||||||
|
base_case = next(case for case in ONE_GPU_CASES if case.id == "flux_image_t2i")
|
||||||
|
|
||||||
|
return replace(
|
||||||
|
base_case,
|
||||||
|
id="flux_image_t2i_layerwise_cpu_offload_5090",
|
||||||
|
server_args=replace(
|
||||||
|
base_case.server_args,
|
||||||
|
dit_layerwise_offload=True,
|
||||||
|
dit_offload_prefetch_size=5,
|
||||||
|
text_encoder_cpu_offload=True,
|
||||||
|
extras=[
|
||||||
|
*base_case.server_args.extras,
|
||||||
|
"--dit-cpu-offload",
|
||||||
|
"--pin-cpu-memory",
|
||||||
|
],
|
||||||
|
),
|
||||||
|
sampling_params=replace(
|
||||||
|
T2I_sampling_params,
|
||||||
|
output_size="512x512",
|
||||||
|
extras={"num_inference_steps": 4, "seed": 0},
|
||||||
|
),
|
||||||
|
run_perf_check=False,
|
||||||
|
run_consistency_check=False,
|
||||||
|
run_component_accuracy_check=False,
|
||||||
|
run_models_api_check=False,
|
||||||
|
run_t2v_input_reference_check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
ONE_GPU_5090_CASES = _select_5090_canary_cases(
|
||||||
|
(
|
||||||
|
"zimage_image_t2i",
|
||||||
|
"flux_2_klein_base_image_t2i",
|
||||||
|
"wan2_1_t2v_1.3b",
|
||||||
|
"turbo_wan2_1_t2v_1.3b",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
ONE_GPU_5090_CASES.append(_make_5090_flux_layerwise_cpu_offload_case())
|
||||||
|
|
||||||
|
|
||||||
def _discover_unit_tests() -> list[str]:
|
def _discover_unit_tests() -> list[str]:
|
||||||
unit_dir = Path(__file__).resolve().parent.parent / "unit"
|
unit_dir = Path(__file__).resolve().parent.parent / "unit"
|
||||||
if not unit_dir.is_dir():
|
if not unit_dir.is_dir():
|
||||||
@@ -879,6 +953,9 @@ PARAMETRIZED_CASE_GROUPS = {
|
|||||||
"1-gpu": [
|
"1-gpu": [
|
||||||
("test_server_1_gpu.py", ONE_GPU_CASES),
|
("test_server_1_gpu.py", ONE_GPU_CASES),
|
||||||
],
|
],
|
||||||
|
"1-gpu-5090": [
|
||||||
|
("test_server_1_gpu_5090.py", ONE_GPU_5090_CASES),
|
||||||
|
],
|
||||||
"2-gpu": [
|
"2-gpu": [
|
||||||
("test_server_2_gpu.py", TWO_GPU_CASES),
|
("test_server_2_gpu.py", TWO_GPU_CASES),
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"model": "Diffusion Server",
|
||||||
|
"hardware": "CI RTX 5090 pool",
|
||||||
|
"description": "Reference numbers captured from real 5090 CI runner history (PR 29791 run 28492141274, job 84450974283).",
|
||||||
|
"last_updated": "2026-07-01"
|
||||||
|
},
|
||||||
|
"tolerances": {
|
||||||
|
"long_term": {
|
||||||
|
"e2e": 0.15,
|
||||||
|
"denoise_stage": 0.1,
|
||||||
|
"non_denoise_stage": 0.5,
|
||||||
|
"denoise_step": 0.25,
|
||||||
|
"denoise_agg": 0.15
|
||||||
|
},
|
||||||
|
"pr_test": {
|
||||||
|
"e2e": 0.25,
|
||||||
|
"denoise_stage": 0.25,
|
||||||
|
"non_denoise_stage": 0.8,
|
||||||
|
"denoise_step": 0.3,
|
||||||
|
"denoise_agg": 0.2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"improvement_reporting": {
|
||||||
|
"threshold": 0.2
|
||||||
|
},
|
||||||
|
"sampling": {
|
||||||
|
"step_fractions": [
|
||||||
|
0.0,
|
||||||
|
0.2,
|
||||||
|
0.4,
|
||||||
|
0.6,
|
||||||
|
0.8,
|
||||||
|
1.0
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"scenarios": {
|
||||||
|
"flux_2_klein_base_image_t2i": {
|
||||||
|
"stages_ms": {
|
||||||
|
"DecodingStage": 18.19,
|
||||||
|
"DenoisingStage": 18213.48,
|
||||||
|
"ImageVAEEncodingStage": 0.01,
|
||||||
|
"InputValidationStage": 0.08,
|
||||||
|
"LatentPreparationStage": 4.87,
|
||||||
|
"TextEncodingStage": 104.46,
|
||||||
|
"TimestepPreparationStage": 142.18
|
||||||
|
},
|
||||||
|
"denoise_step_ms": {
|
||||||
|
"0": 219.58,
|
||||||
|
"10": 359.79,
|
||||||
|
"20": 361.55,
|
||||||
|
"29": 361.99,
|
||||||
|
"39": 362.57,
|
||||||
|
"49": 362.89
|
||||||
|
},
|
||||||
|
"expected_e2e_ms": 18641.09,
|
||||||
|
"expected_avg_denoise_ms": 357.93,
|
||||||
|
"expected_median_denoise_ms": 361.8,
|
||||||
|
"estimated_full_test_time_s": 94.0
|
||||||
|
},
|
||||||
|
"wan2_1_t2v_1.3b": {
|
||||||
|
"stages_ms": {
|
||||||
|
"DecodingStage": 1202.43,
|
||||||
|
"DenoisingStage": 21452.59,
|
||||||
|
"InputValidationStage": 0.09,
|
||||||
|
"LatentPreparationStage": 0.23,
|
||||||
|
"TextEncodingStage": 1210.22,
|
||||||
|
"TimestepPreparationStage": 3.91
|
||||||
|
},
|
||||||
|
"denoise_step_ms": {
|
||||||
|
"0": 480.39,
|
||||||
|
"10": 426.11,
|
||||||
|
"20": 427.32,
|
||||||
|
"29": 428.09,
|
||||||
|
"39": 428.04,
|
||||||
|
"49": 422.31
|
||||||
|
},
|
||||||
|
"expected_e2e_ms": 23877.73,
|
||||||
|
"expected_avg_denoise_ms": 428.74,
|
||||||
|
"expected_median_denoise_ms": 427.94,
|
||||||
|
"estimated_full_test_time_s": 160.9
|
||||||
|
},
|
||||||
|
"zimage_image_t2i": {
|
||||||
|
"stages_ms": {
|
||||||
|
"DecodingStage": 7.11,
|
||||||
|
"DenoisingStage": 2229.48,
|
||||||
|
"InputValidationStage": 0.04,
|
||||||
|
"LatentPreparationStage": 0.17,
|
||||||
|
"TextEncodingStage": 252.73,
|
||||||
|
"TimestepPreparationStage": 38.57
|
||||||
|
},
|
||||||
|
"denoise_step_ms": {
|
||||||
|
"0": 39.34,
|
||||||
|
"2": 277.35,
|
||||||
|
"3": 267.94,
|
||||||
|
"5": 279.01,
|
||||||
|
"6": 273.01,
|
||||||
|
"8": 275.12
|
||||||
|
},
|
||||||
|
"expected_e2e_ms": 2533.95,
|
||||||
|
"expected_avg_denoise_ms": 246.97,
|
||||||
|
"expected_median_denoise_ms": 273.01,
|
||||||
|
"estimated_full_test_time_s": 329.8
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"model": "Diffusion Server",
|
||||||
|
"hardware": "CI B200 pool",
|
||||||
|
"description": "Reference estimates for B200-only diffusion cases, split out from the shared diffusion baseline file.",
|
||||||
|
"last_updated": "2026-07-01"
|
||||||
|
},
|
||||||
|
"tolerances": {
|
||||||
|
"long_term": {
|
||||||
|
"e2e": 0.15,
|
||||||
|
"denoise_stage": 0.1,
|
||||||
|
"non_denoise_stage": 0.5,
|
||||||
|
"denoise_step": 0.25,
|
||||||
|
"denoise_agg": 0.15
|
||||||
|
},
|
||||||
|
"pr_test": {
|
||||||
|
"e2e": 0.25,
|
||||||
|
"denoise_stage": 0.25,
|
||||||
|
"non_denoise_stage": 0.8,
|
||||||
|
"denoise_step": 0.3,
|
||||||
|
"denoise_agg": 0.2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"improvement_reporting": {
|
||||||
|
"threshold": 0.2
|
||||||
|
},
|
||||||
|
"sampling": {
|
||||||
|
"step_fractions": [
|
||||||
|
0.0,
|
||||||
|
0.2,
|
||||||
|
0.4,
|
||||||
|
0.6,
|
||||||
|
0.8,
|
||||||
|
1.0
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"scenarios": {
|
||||||
|
"flux1_modelopt_nvfp4_t2i": {
|
||||||
|
"stages_ms": {},
|
||||||
|
"denoise_step_ms": {},
|
||||||
|
"expected_e2e_ms": 0.0,
|
||||||
|
"expected_avg_denoise_ms": 0.0,
|
||||||
|
"expected_median_denoise_ms": 0.0,
|
||||||
|
"estimated_full_test_time_s": 71.2
|
||||||
|
},
|
||||||
|
"flux2_modelopt_nvfp4_t2i": {
|
||||||
|
"stages_ms": {},
|
||||||
|
"denoise_step_ms": {},
|
||||||
|
"expected_e2e_ms": 0.0,
|
||||||
|
"expected_avg_denoise_ms": 0.0,
|
||||||
|
"expected_median_denoise_ms": 0.0,
|
||||||
|
"estimated_full_test_time_s": 592.3
|
||||||
|
},
|
||||||
|
"qwen_image_2512_modelopt_nvfp4_t2i": {
|
||||||
|
"stages_ms": {},
|
||||||
|
"denoise_step_ms": {},
|
||||||
|
"expected_e2e_ms": 0.0,
|
||||||
|
"expected_avg_denoise_ms": 0.0,
|
||||||
|
"expected_median_denoise_ms": 0.0,
|
||||||
|
"estimated_full_test_time_s": 120.0
|
||||||
|
},
|
||||||
|
"wan22_modelopt_nvfp4_t2v": {
|
||||||
|
"stages_ms": {},
|
||||||
|
"denoise_step_ms": {},
|
||||||
|
"expected_e2e_ms": 0.0,
|
||||||
|
"expected_avg_denoise_ms": 0.0,
|
||||||
|
"expected_median_denoise_ms": 0.0,
|
||||||
|
"estimated_full_test_time_s": 181.8
|
||||||
|
},
|
||||||
|
"ideogram4_nvfp4_t2i": {
|
||||||
|
"stages_ms": {},
|
||||||
|
"denoise_step_ms": {},
|
||||||
|
"expected_e2e_ms": 0.0,
|
||||||
|
"expected_avg_denoise_ms": 0.0,
|
||||||
|
"expected_median_denoise_ms": 0.0,
|
||||||
|
"estimated_full_test_time_s": 300.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
-32
@@ -2829,38 +2829,6 @@
|
|||||||
"expected_avg_denoise_ms": 0.0,
|
"expected_avg_denoise_ms": 0.0,
|
||||||
"expected_median_denoise_ms": 0.0,
|
"expected_median_denoise_ms": 0.0,
|
||||||
"estimated_full_test_time_s": 73.7
|
"estimated_full_test_time_s": 73.7
|
||||||
},
|
|
||||||
"flux1_modelopt_nvfp4_t2i": {
|
|
||||||
"stages_ms": {},
|
|
||||||
"denoise_step_ms": {},
|
|
||||||
"expected_e2e_ms": 0.0,
|
|
||||||
"expected_avg_denoise_ms": 0.0,
|
|
||||||
"expected_median_denoise_ms": 0.0,
|
|
||||||
"estimated_full_test_time_s": 71.2
|
|
||||||
},
|
|
||||||
"flux2_modelopt_nvfp4_t2i": {
|
|
||||||
"stages_ms": {},
|
|
||||||
"denoise_step_ms": {},
|
|
||||||
"expected_e2e_ms": 0.0,
|
|
||||||
"expected_avg_denoise_ms": 0.0,
|
|
||||||
"expected_median_denoise_ms": 0.0,
|
|
||||||
"estimated_full_test_time_s": 592.3
|
|
||||||
},
|
|
||||||
"qwen_image_2512_modelopt_nvfp4_t2i": {
|
|
||||||
"stages_ms": {},
|
|
||||||
"denoise_step_ms": {},
|
|
||||||
"expected_e2e_ms": 0.0,
|
|
||||||
"expected_avg_denoise_ms": 0.0,
|
|
||||||
"expected_median_denoise_ms": 0.0,
|
|
||||||
"estimated_full_test_time_s": 120.0
|
|
||||||
},
|
|
||||||
"wan22_modelopt_nvfp4_t2v": {
|
|
||||||
"stages_ms": {},
|
|
||||||
"denoise_step_ms": {},
|
|
||||||
"expected_e2e_ms": 0.0,
|
|
||||||
"expected_avg_denoise_ms": 0.0,
|
|
||||||
"expected_median_denoise_ms": 0.0,
|
|
||||||
"estimated_full_test_time_s": 181.8
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
"""
|
||||||
|
Config-driven diffusion canary tests for the 1-GPU 5090 PR runner.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from sglang.multimodal_gen.test.server.common.case_fixtures import (
|
||||||
|
diffusion_case_fixture,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.test.server.gpu_cases import ONE_GPU_5090_CASES
|
||||||
|
from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401
|
||||||
|
DiffusionServerBase,
|
||||||
|
diffusion_server,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDiffusionServerOneGpu5090(DiffusionServerBase):
|
||||||
|
"""Canary tests for lightweight 1-GPU diffusion cases on 5090."""
|
||||||
|
|
||||||
|
case = diffusion_case_fixture(ONE_GPU_5090_CASES)
|
||||||
@@ -41,6 +41,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
|
|||||||
PerformanceSummary,
|
PerformanceSummary,
|
||||||
ScenarioConfig,
|
ScenarioConfig,
|
||||||
get_model_task_type_for_server_args,
|
get_model_task_type_for_server_args,
|
||||||
|
get_perf_baseline_path,
|
||||||
)
|
)
|
||||||
from sglang.multimodal_gen.test.test_utils import (
|
from sglang.multimodal_gen.test.test_utils import (
|
||||||
SGL_TEST_FILES_CI_DATA_REVISION,
|
SGL_TEST_FILES_CI_DATA_REVISION,
|
||||||
@@ -50,6 +51,7 @@ from sglang.multimodal_gen.test.test_utils import (
|
|||||||
extract_key_frames_from_video,
|
extract_key_frames_from_video,
|
||||||
get_consistency_gt_candidates,
|
get_consistency_gt_candidates,
|
||||||
get_consistency_gt_remote_files,
|
get_consistency_gt_remote_files,
|
||||||
|
get_consistency_threshold_path,
|
||||||
get_consistency_thresholds,
|
get_consistency_thresholds,
|
||||||
get_dynamic_server_port,
|
get_dynamic_server_port,
|
||||||
gt_exists,
|
gt_exists,
|
||||||
@@ -232,7 +234,7 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext:
|
|||||||
logger.error(
|
logger.error(
|
||||||
f'\n{"=" * 60}\n'
|
f'\n{"=" * 60}\n'
|
||||||
f'Add "estimated_full_test_time_s" to scenario "{case.id}":\n\n'
|
f'Add "estimated_full_test_time_s" to scenario "{case.id}":\n\n'
|
||||||
f"File: python/sglang/multimodal_gen/test/server/perf_baselines.json\n\n"
|
f"File: {get_perf_baseline_path()}\n\n"
|
||||||
f' "{case.id}": {{\n'
|
f' "{case.id}": {{\n'
|
||||||
f" ...\n"
|
f" ...\n"
|
||||||
f' "estimated_full_test_time_s": {_measured_full_time:.1f}\n'
|
f' "estimated_full_test_time_s": {_measured_full_time:.1f}\n'
|
||||||
@@ -441,7 +443,7 @@ class DiffusionServerBase:
|
|||||||
self._dump_baseline_for_testcase(case, summary, missing_scenario)
|
self._dump_baseline_for_testcase(case, summary, missing_scenario)
|
||||||
if missing_scenario:
|
if missing_scenario:
|
||||||
pytest.fail(
|
pytest.fail(
|
||||||
f"Testcase '{case.id}' not found in perf_baselines.json"
|
f"Testcase '{case.id}' not found in {get_perf_baseline_path()}"
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -552,7 +554,7 @@ class DiffusionServerBase:
|
|||||||
)
|
)
|
||||||
action = "add" if missing_scenario else "update"
|
action = "add" if missing_scenario else "update"
|
||||||
output = f"""
|
output = f"""
|
||||||
{action} this baseline in the "scenarios" section of perf_baselines.json:
|
{action} this baseline in the "scenarios" section of {get_perf_baseline_path()}:
|
||||||
|
|
||||||
"{case.id}": {json.dumps(baseline, indent=4)}
|
"{case.id}": {json.dumps(baseline, indent=4)}
|
||||||
|
|
||||||
@@ -607,10 +609,10 @@ Add the expected file(s) to sgl-project/ci-data in diffusion-ci/consistency_gt/s
|
|||||||
|
|
||||||
For this case, expected file(s): {names}
|
For this case, expected file(s): {names}
|
||||||
|
|
||||||
Repository: https://github.com/sgl-project/ci-data (path: diffusion-ci/consistency_gt/sglang_generated/)
|
Repository: https://github.com/sgl-project/ci-data (path: diffusion-ci/consistency_gt/sglang_generated/, with optional platform subdirectories such as 5090/)
|
||||||
Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
||||||
|
|
||||||
(Optional) Per-case override in consistency_threshold.json:
|
(Optional) Per-case override in {get_consistency_threshold_path()}:
|
||||||
"cases": {{
|
"cases": {{
|
||||||
"{case.id}": {{
|
"{case.id}": {{
|
||||||
"clip_threshold": 0.92,
|
"clip_threshold": 0.92,
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ pytest python/sglang/multimodal_gen/test/server/test_server_1_gpu.py -k qwen_ima
|
|||||||
To add a new testcase:
|
To add a new testcase:
|
||||||
1. add your testcase with case-id: `my_new_test_case_id` to `ONE_GPU_CASES`, `ONE_GPU_MODELOPT_FP8_CASES`, `ONE_GPU_B200_CASES`, or `TWO_GPU_CASES`
|
1. add your testcase with case-id: `my_new_test_case_id` to `ONE_GPU_CASES`, `ONE_GPU_MODELOPT_FP8_CASES`, `ONE_GPU_B200_CASES`, or `TWO_GPU_CASES`
|
||||||
2. run `SGLANG_GEN_BASELINE=1 pytest -s python/sglang/multimodal_gen/test/server/ -k my_new_test_case_id`
|
2. run `SGLANG_GEN_BASELINE=1 pytest -s python/sglang/multimodal_gen/test/server/ -k my_new_test_case_id`
|
||||||
3. insert or override the corresponding scenario in `scenarios` section of perf_baselines.json with the output baseline of step-2
|
3. insert or override the corresponding scenario in the platform JSON under `perf_baselines/`
|
||||||
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
@@ -33,6 +33,7 @@ from sglang.multimodal_gen.registry import (
|
|||||||
get_model_info,
|
get_model_info,
|
||||||
get_pipeline_config_classes,
|
get_pipeline_config_classes,
|
||||||
)
|
)
|
||||||
|
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||||
from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord
|
from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord
|
||||||
|
|
||||||
|
|
||||||
@@ -657,6 +658,57 @@ MODELOPT_WAN22_NVFP4_MODEL = "nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4"
|
|||||||
MODELOPT_NVFP4_B200_ENV_VARS = {}
|
MODELOPT_NVFP4_B200_ENV_VARS = {}
|
||||||
MODELOPT_WAN22_NVFP4_B200_ENV_VARS = {}
|
MODELOPT_WAN22_NVFP4_B200_ENV_VARS = {}
|
||||||
|
|
||||||
|
PERF_BASELINE_PLATFORM_ENV = "SGLANG_DIFFUSION_PERF_BASELINE_PLATFORM"
|
||||||
|
PERF_BASELINE_DIR = Path(__file__).with_name("perf_baselines")
|
||||||
|
PERF_BASELINE_FILE_BY_PLATFORM = {
|
||||||
|
"h100": "h100.json",
|
||||||
|
"b200": "b200.json",
|
||||||
|
"5090": "5090.json",
|
||||||
|
}
|
||||||
|
PERF_BASELINE_PLATFORM_ALIASES = {
|
||||||
|
"sm90": "h100",
|
||||||
|
"hopper": "h100",
|
||||||
|
"h100": "h100",
|
||||||
|
"sm100": "b200",
|
||||||
|
"blackwell": "b200",
|
||||||
|
"b200": "b200",
|
||||||
|
"sm120": "5090",
|
||||||
|
"rtx5090": "5090",
|
||||||
|
"5090": "5090",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_perf_baseline_platform(platform: str) -> str:
|
||||||
|
normalized = platform.strip().lower().replace("_", "-")
|
||||||
|
normalized = normalized.replace("-", "")
|
||||||
|
if normalized not in PERF_BASELINE_PLATFORM_ALIASES:
|
||||||
|
valid = ", ".join(sorted(PERF_BASELINE_FILE_BY_PLATFORM))
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid diffusion perf baseline platform {platform!r}. "
|
||||||
|
f"Expected one of: {valid}"
|
||||||
|
)
|
||||||
|
return PERF_BASELINE_PLATFORM_ALIASES[normalized]
|
||||||
|
|
||||||
|
|
||||||
|
def get_perf_baseline_platform() -> str:
|
||||||
|
override = os.getenv(PERF_BASELINE_PLATFORM_ENV)
|
||||||
|
if override:
|
||||||
|
return _normalize_perf_baseline_platform(override)
|
||||||
|
if current_platform.is_sm120():
|
||||||
|
return "5090"
|
||||||
|
if current_platform.is_blackwell():
|
||||||
|
return "b200"
|
||||||
|
return "h100"
|
||||||
|
|
||||||
|
|
||||||
|
def get_perf_baseline_path(platform: str | None = None) -> Path:
|
||||||
|
baseline_platform = (
|
||||||
|
_normalize_perf_baseline_platform(platform)
|
||||||
|
if platform is not None
|
||||||
|
else get_perf_baseline_platform()
|
||||||
|
)
|
||||||
|
return PERF_BASELINE_DIR / PERF_BASELINE_FILE_BY_PLATFORM[baseline_platform]
|
||||||
|
|
||||||
|
|
||||||
def _make_modelopt_ci_case(
|
def _make_modelopt_ci_case(
|
||||||
case_id: str,
|
case_id: str,
|
||||||
@@ -694,7 +746,7 @@ def _with_default_num_gpus(
|
|||||||
|
|
||||||
# Load global configuration
|
# Load global configuration
|
||||||
BASELINE_CONFIG = (
|
BASELINE_CONFIG = (
|
||||||
BaselineConfig.load(Path(__file__).with_name("perf_baselines.json"))
|
BaselineConfig.load(get_perf_baseline_path())
|
||||||
.update(Path(__file__).parent / "ascend" / "perf_baselines_npu.json")
|
.update(Path(__file__).parent / "ascend" / "perf_baselines_npu.json")
|
||||||
.update(Path(__file__).parent / "musa" / "perf_baselines_musa.json")
|
.update(Path(__file__).parent / "musa" / "perf_baselines_musa.json")
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
SGL_TEST_FILES_CI_DATA_REVISION = "4a271ef34602043f19d253f0d30a5f653fe11325"
|
SGL_TEST_FILES_CI_DATA_REVISION = "702d939e23f17b42183329dace60f221d2587056"
|
||||||
|
|
||||||
if current_platform.is_npu():
|
if current_platform.is_npu():
|
||||||
SGL_TEST_FILES_CI_DATA_REVISION = "670d66a8a290b62c0c3c077b3e9b0f4a4d9a44e7"
|
SGL_TEST_FILES_CI_DATA_REVISION = "670d66a8a290b62c0c3c077b3e9b0f4a4d9a44e7"
|
||||||
@@ -79,9 +79,26 @@ SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_CASES = frozenset(
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
CONSISTENCY_THRESHOLD_JSON_PATH = (
|
CONSISTENCY_PLATFORM_ENV = "SGLANG_DIFFUSION_CONSISTENCY_PLATFORM"
|
||||||
Path(__file__).resolve().parent / "server" / "consistency_threshold.json"
|
CONSISTENCY_THRESHOLD_DIR = (
|
||||||
|
Path(__file__).resolve().parent / "server" / "consistency_thresholds"
|
||||||
)
|
)
|
||||||
|
CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM = {
|
||||||
|
"h100": "h100.json",
|
||||||
|
"b200": "b200.json",
|
||||||
|
"5090": "5090.json",
|
||||||
|
}
|
||||||
|
CONSISTENCY_PLATFORM_ALIASES = {
|
||||||
|
"sm90": "h100",
|
||||||
|
"hopper": "h100",
|
||||||
|
"h100": "h100",
|
||||||
|
"sm100": "b200",
|
||||||
|
"blackwell": "b200",
|
||||||
|
"b200": "b200",
|
||||||
|
"sm120": "5090",
|
||||||
|
"rtx5090": "5090",
|
||||||
|
"5090": "5090",
|
||||||
|
}
|
||||||
CLIP_MODEL_NAME = "openai/clip-vit-large-patch14"
|
CLIP_MODEL_NAME = "openai/clip-vit-large-patch14"
|
||||||
DEFAULT_CLIP_THRESHOLD_IMAGE = 0.92
|
DEFAULT_CLIP_THRESHOLD_IMAGE = 0.92
|
||||||
DEFAULT_CLIP_THRESHOLD_VIDEO = 0.90
|
DEFAULT_CLIP_THRESHOLD_VIDEO = 0.90
|
||||||
@@ -694,14 +711,74 @@ def validate_video_file(
|
|||||||
), f"Video height mismatch: expected {expected_height}, got {actual_height}"
|
), f"Video height mismatch: expected {expected_height}, got {actual_height}"
|
||||||
|
|
||||||
|
|
||||||
def _load_threshold_json() -> dict[str, Any]:
|
def _normalize_consistency_platform(platform: str) -> str:
|
||||||
"""Load consistency_threshold.json; returns {} if missing."""
|
normalized = platform.strip().lower().replace("_", "-")
|
||||||
if not CONSISTENCY_THRESHOLD_JSON_PATH.exists():
|
normalized = normalized.replace("-", "")
|
||||||
|
if normalized not in CONSISTENCY_PLATFORM_ALIASES:
|
||||||
|
valid = ", ".join(sorted(CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM))
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid diffusion consistency platform {platform!r}. "
|
||||||
|
f"Expected one of: {valid}"
|
||||||
|
)
|
||||||
|
return CONSISTENCY_PLATFORM_ALIASES[normalized]
|
||||||
|
|
||||||
|
|
||||||
|
def get_consistency_platform() -> str:
|
||||||
|
override = os.getenv(CONSISTENCY_PLATFORM_ENV)
|
||||||
|
if override:
|
||||||
|
return _normalize_consistency_platform(override)
|
||||||
|
if current_platform.is_sm120():
|
||||||
|
return "5090"
|
||||||
|
if current_platform.is_blackwell():
|
||||||
|
return "b200"
|
||||||
|
return "h100"
|
||||||
|
|
||||||
|
|
||||||
|
def get_consistency_threshold_path(platform: str | None = None) -> Path:
|
||||||
|
threshold_platform = (
|
||||||
|
_normalize_consistency_platform(platform)
|
||||||
|
if platform is not None
|
||||||
|
else get_consistency_platform()
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
CONSISTENCY_THRESHOLD_DIR
|
||||||
|
/ CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM[threshold_platform]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_threshold_file(path: Path) -> dict[str, Any]:
|
||||||
|
if not path.exists():
|
||||||
return {}
|
return {}
|
||||||
with CONSISTENCY_THRESHOLD_JSON_PATH.open("r", encoding="utf-8") as f:
|
with path.open("r", encoding="utf-8") as f:
|
||||||
return json.load(f)
|
return json.load(f)
|
||||||
|
|
||||||
|
|
||||||
|
def _merge_threshold_metadata(
|
||||||
|
base: dict[str, Any], override: dict[str, Any]
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
merged = dict(base)
|
||||||
|
if "cases" in base or "cases" in override:
|
||||||
|
merged["cases"] = {
|
||||||
|
**base.get("cases", {}),
|
||||||
|
**override.get("cases", {}),
|
||||||
|
}
|
||||||
|
for key, value in override.items():
|
||||||
|
if key != "cases":
|
||||||
|
merged[key] = value
|
||||||
|
return merged
|
||||||
|
|
||||||
|
|
||||||
|
def _load_threshold_json() -> dict[str, Any]:
|
||||||
|
metadata = _load_threshold_file(get_consistency_threshold_path("h100"))
|
||||||
|
platform = get_consistency_platform()
|
||||||
|
if platform == "h100":
|
||||||
|
return metadata
|
||||||
|
return _merge_threshold_metadata(
|
||||||
|
metadata,
|
||||||
|
_load_threshold_file(get_consistency_threshold_path(platform)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ConsistencyThresholds:
|
class ConsistencyThresholds:
|
||||||
clip_threshold: float
|
clip_threshold: float
|
||||||
@@ -958,10 +1035,9 @@ def _consistency_gt_filenames(
|
|||||||
return [f"{case_id}_{n}gpu.{ext}"]
|
return [f"{case_id}_{n}gpu.{ext}"]
|
||||||
|
|
||||||
|
|
||||||
def get_consistency_gt_candidates(
|
def _base_consistency_gt_candidates(
|
||||||
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
|
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
"""Return candidate GT filenames for local consistency data."""
|
|
||||||
n = num_gpus
|
n = num_gpus
|
||||||
if is_video:
|
if is_video:
|
||||||
return [
|
return [
|
||||||
@@ -975,6 +1051,31 @@ def get_consistency_gt_candidates(
|
|||||||
return [f"{base}.{e}" for e in exts]
|
return [f"{base}.{e}" for e in exts]
|
||||||
|
|
||||||
|
|
||||||
|
def get_consistency_gt_candidate_sets(
|
||||||
|
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
|
||||||
|
) -> list[list[str]]:
|
||||||
|
candidates = _base_consistency_gt_candidates(
|
||||||
|
case_id, num_gpus, is_video, output_format
|
||||||
|
)
|
||||||
|
platform = get_consistency_platform()
|
||||||
|
if platform == "h100":
|
||||||
|
return [candidates]
|
||||||
|
return [[f"{platform}/{candidate}" for candidate in candidates], candidates]
|
||||||
|
|
||||||
|
|
||||||
|
def get_consistency_gt_candidates(
|
||||||
|
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
|
||||||
|
) -> list[str]:
|
||||||
|
"""Return candidate GT filenames for local consistency data."""
|
||||||
|
return [
|
||||||
|
candidate
|
||||||
|
for candidate_set in get_consistency_gt_candidate_sets(
|
||||||
|
case_id, num_gpus, is_video, output_format
|
||||||
|
)
|
||||||
|
for candidate in candidate_set
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def get_consistency_gt_remote_files(
|
def get_consistency_gt_remote_files(
|
||||||
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
|
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
|
||||||
) -> list[tuple[str, str]]:
|
) -> list[tuple[str, str]]:
|
||||||
@@ -1003,6 +1104,21 @@ def _remote_consistency_gt_candidates(
|
|||||||
return [(filename, f"{base_url}/{filename}") for filename in filenames]
|
return [(filename, f"{base_url}/{filename}") for filename in filenames]
|
||||||
|
|
||||||
|
|
||||||
|
def _remote_consistency_gt_candidate_sets(
|
||||||
|
base_url: str,
|
||||||
|
case_id: str,
|
||||||
|
num_gpus: int,
|
||||||
|
is_video: bool,
|
||||||
|
output_format: str | None = None,
|
||||||
|
) -> list[list[tuple[str, str]]]:
|
||||||
|
return [
|
||||||
|
[(filename, f"{base_url}/{filename}") for filename in filenames]
|
||||||
|
for filenames in get_consistency_gt_candidate_sets(
|
||||||
|
case_id, num_gpus, is_video, output_format
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def _is_ascend_consistency_case(case_id: str) -> bool:
|
def _is_ascend_consistency_case(case_id: str) -> bool:
|
||||||
return "npu" in case_id
|
return "npu" in case_id
|
||||||
|
|
||||||
@@ -1086,14 +1202,15 @@ def _find_remote_consistency_gt_files(
|
|||||||
# Avoid accidentally comparing non-comparable CI cases against official GT.
|
# Avoid accidentally comparing non-comparable CI cases against official GT.
|
||||||
bases = (SGL_TEST_FILES_CONSISTENCY_GT_BASE,)
|
bases = (SGL_TEST_FILES_CONSISTENCY_GT_BASE,)
|
||||||
for base_url in bases:
|
for base_url in bases:
|
||||||
candidates = _remote_consistency_gt_candidates(
|
candidate_sets = _remote_consistency_gt_candidate_sets(
|
||||||
base_url, case_id, num_gpus, is_video, output_format
|
base_url, case_id, num_gpus, is_video, output_format
|
||||||
)
|
)
|
||||||
if is_video:
|
for candidates in candidate_sets:
|
||||||
exists = [_remote_file_exists(url) for _, url in candidates]
|
if is_video:
|
||||||
if all(status is not False for status in exists):
|
exists = [_remote_file_exists(url) for _, url in candidates]
|
||||||
return candidates
|
if all(status is not False for status in exists):
|
||||||
else:
|
return candidates
|
||||||
|
continue
|
||||||
uncertain_candidate = None
|
uncertain_candidate = None
|
||||||
for filename, url in candidates:
|
for filename, url in candidates:
|
||||||
exists = _remote_file_exists(url)
|
exists = _remote_file_exists(url)
|
||||||
@@ -1122,7 +1239,8 @@ def _get_consistency_gt_cache_key(
|
|||||||
) -> str:
|
) -> str:
|
||||||
gt_dir = _get_consistency_gt_dir()
|
gt_dir = _get_consistency_gt_dir()
|
||||||
source = str(gt_dir) if gt_dir is not None else "remote"
|
source = str(gt_dir) if gt_dir is not None else "remote"
|
||||||
return f"{case_id}:{num_gpus}:{is_video}:{output_format or ''}:{source}"
|
platform = get_consistency_platform()
|
||||||
|
return f"{platform}:{case_id}:{num_gpus}:{is_video}:{output_format or ''}:{source}"
|
||||||
|
|
||||||
|
|
||||||
def load_consistency_gt(
|
def load_consistency_gt(
|
||||||
@@ -1139,29 +1257,43 @@ def load_consistency_gt(
|
|||||||
if cached is not None:
|
if cached is not None:
|
||||||
return cached
|
return cached
|
||||||
|
|
||||||
filenames = _consistency_gt_filenames(case_id, num_gpus, is_video, output_format)
|
|
||||||
images: list[np.ndarray] = []
|
images: list[np.ndarray] = []
|
||||||
|
|
||||||
gt_dir = _get_consistency_gt_dir()
|
gt_dir = _get_consistency_gt_dir()
|
||||||
if gt_dir is not None:
|
if gt_dir is not None:
|
||||||
candidates = get_consistency_gt_candidates(
|
candidate_sets = get_consistency_gt_candidate_sets(
|
||||||
case_id, num_gpus, is_video, output_format
|
case_id, num_gpus, is_video, output_format
|
||||||
)
|
)
|
||||||
if is_video:
|
if is_video:
|
||||||
for fn in candidates:
|
selected = None
|
||||||
path = gt_dir / fn
|
for candidates in candidate_sets:
|
||||||
if not path.exists():
|
if all((gt_dir / fn).exists() for fn in candidates):
|
||||||
raise FileNotFoundError(f"GT image not found: {path}")
|
selected = candidates
|
||||||
arr = np.array(Image.open(path).convert("RGB"))
|
break
|
||||||
images.append(arr)
|
if selected is None:
|
||||||
|
tried = ", ".join(
|
||||||
|
candidate
|
||||||
|
for candidates in candidate_sets
|
||||||
|
for candidate in candidates
|
||||||
|
)
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"GT images not found in {gt_dir}. Tried: {tried}"
|
||||||
|
)
|
||||||
|
for fn in selected:
|
||||||
|
images.append(np.array(Image.open(gt_dir / fn).convert("RGB")))
|
||||||
else:
|
else:
|
||||||
path = None
|
path = None
|
||||||
for fn in candidates:
|
for fn in get_consistency_gt_candidates(
|
||||||
|
case_id, num_gpus, is_video, output_format
|
||||||
|
):
|
||||||
candidate = gt_dir / fn
|
candidate = gt_dir / fn
|
||||||
if candidate.exists():
|
if candidate.exists():
|
||||||
path = candidate
|
path = candidate
|
||||||
break
|
break
|
||||||
if path is None:
|
if path is None:
|
||||||
|
candidates = get_consistency_gt_candidates(
|
||||||
|
case_id, num_gpus, is_video, output_format
|
||||||
|
)
|
||||||
raise FileNotFoundError(
|
raise FileNotFoundError(
|
||||||
f"GT image not found in {gt_dir}. Tried: {', '.join(candidates)}"
|
f"GT image not found in {gt_dir}. Tried: {', '.join(candidates)}"
|
||||||
)
|
)
|
||||||
@@ -1172,8 +1304,11 @@ def load_consistency_gt(
|
|||||||
case_id, num_gpus, is_video, output_format
|
case_id, num_gpus, is_video, output_format
|
||||||
)
|
)
|
||||||
if not remote_files:
|
if not remote_files:
|
||||||
|
candidates = get_consistency_gt_candidates(
|
||||||
|
case_id, num_gpus, is_video, output_format
|
||||||
|
)
|
||||||
raise FileNotFoundError(
|
raise FileNotFoundError(
|
||||||
f"GT image not found for {case_id}. Tried: {', '.join(filenames)}"
|
f"GT image not found for {case_id}. Tried: {', '.join(candidates)}"
|
||||||
)
|
)
|
||||||
for _, url in remote_files:
|
for _, url in remote_files:
|
||||||
images.append(_load_remote_gt_image(url))
|
images.append(_load_remote_gt_image(url))
|
||||||
@@ -1210,12 +1345,19 @@ def gt_exists(
|
|||||||
"""Check whether GT image(s) exist."""
|
"""Check whether GT image(s) exist."""
|
||||||
gt_dir = _get_consistency_gt_dir()
|
gt_dir = _get_consistency_gt_dir()
|
||||||
if gt_dir is not None:
|
if gt_dir is not None:
|
||||||
candidates = get_consistency_gt_candidates(
|
candidate_sets = get_consistency_gt_candidate_sets(
|
||||||
case_id, num_gpus, is_video, output_format
|
case_id, num_gpus, is_video, output_format
|
||||||
)
|
)
|
||||||
if is_video:
|
if is_video:
|
||||||
return all((gt_dir / c).exists() for c in candidates)
|
return any(
|
||||||
return any((gt_dir / c).exists() for c in candidates)
|
all((gt_dir / candidate).exists() for candidate in candidate_set)
|
||||||
|
for candidate_set in candidate_sets
|
||||||
|
)
|
||||||
|
return any(
|
||||||
|
(gt_dir / candidate).exists()
|
||||||
|
for candidate_set in candidate_sets
|
||||||
|
for candidate in candidate_set
|
||||||
|
)
|
||||||
|
|
||||||
cache_key = _get_consistency_gt_cache_key(
|
cache_key = _get_consistency_gt_cache_key(
|
||||||
case_id, num_gpus, is_video, output_format
|
case_id, num_gpus, is_video, output_format
|
||||||
|
|||||||
@@ -54,6 +54,53 @@ def test_remote_video_gt_candidates_survive_inconclusive_probe(monkeypatch):
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_platform_gt_candidates_prefer_platform_then_default(monkeypatch):
|
||||||
|
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "5090")
|
||||||
|
|
||||||
|
assert test_utils.get_consistency_gt_candidates(
|
||||||
|
"unit_image",
|
||||||
|
1,
|
||||||
|
is_video=False,
|
||||||
|
output_format="png",
|
||||||
|
) == [
|
||||||
|
"5090/unit_image_1gpu.png",
|
||||||
|
"5090/unit_image_1gpu.jpg",
|
||||||
|
"5090/unit_image_1gpu.webp",
|
||||||
|
"unit_image_1gpu.png",
|
||||||
|
"unit_image_1gpu.jpg",
|
||||||
|
"unit_image_1gpu.webp",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_threshold_metadata_merges_platform_override():
|
||||||
|
metadata = test_utils._merge_threshold_metadata(
|
||||||
|
{
|
||||||
|
"cases": {
|
||||||
|
"case_a": {
|
||||||
|
"clip_threshold": 0.9,
|
||||||
|
"ssim_threshold": 0.9,
|
||||||
|
"psnr_threshold": 20.0,
|
||||||
|
"mean_abs_diff_threshold": 10.0,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"default_clip_threshold_image": 0.92,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cases": {
|
||||||
|
"case_a": {
|
||||||
|
"clip_threshold": 0.8,
|
||||||
|
"ssim_threshold": 0.7,
|
||||||
|
"psnr_threshold": 12.0,
|
||||||
|
"mean_abs_diff_threshold": 20.0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert metadata["default_clip_threshold_image"] == 0.92
|
||||||
|
assert metadata["cases"]["case_a"]["psnr_threshold"] == 12.0
|
||||||
|
|
||||||
|
|
||||||
def test_pixel_metrics_identical_image():
|
def test_pixel_metrics_identical_image():
|
||||||
image = _solid_image(128)
|
image = _solid_image(128)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from sglang.multimodal_gen.runtime.layers.attention.turbo_layer import (
|
||||||
|
_resolve_turbo_wan_sparse_backend,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum
|
||||||
|
|
||||||
|
|
||||||
|
class TestTurboWanBackendSelection(unittest.TestCase):
|
||||||
|
def test_non_sparse_requested_backend_falls_back_to_attention_type(self):
|
||||||
|
selected, warning = _resolve_turbo_wan_sparse_backend(
|
||||||
|
attention_type="sla",
|
||||||
|
requested_attention_backend="fa",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(selected, AttentionBackendEnum.SLA_ATTN)
|
||||||
|
self.assertIsNotNone(warning)
|
||||||
|
self.assertIn("TurboWan only supports", warning)
|
||||||
|
self.assertIn("attention_backend='fa'", warning)
|
||||||
|
|
||||||
|
def test_sagesla_attention_type_prefers_sage_sparse_backend(self):
|
||||||
|
selected, warning = _resolve_turbo_wan_sparse_backend(
|
||||||
|
attention_type="sagesla",
|
||||||
|
requested_attention_backend="torch_sdpa",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN)
|
||||||
|
self.assertIsNotNone(warning)
|
||||||
|
|
||||||
|
def test_requested_sparse_backend_is_honored(self):
|
||||||
|
selected, warning = _resolve_turbo_wan_sparse_backend(
|
||||||
|
attention_type="sla",
|
||||||
|
requested_attention_backend="sage_sla_attn",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN)
|
||||||
|
self.assertIsNone(warning)
|
||||||
|
|
||||||
|
def test_supported_backend_filter_is_respected(self):
|
||||||
|
selected, warning = _resolve_turbo_wan_sparse_backend(
|
||||||
|
attention_type="sla",
|
||||||
|
requested_attention_backend=None,
|
||||||
|
supported_attention_backends={AttentionBackendEnum.SAGE_SLA_ATTN},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN)
|
||||||
|
self.assertIsNone(warning)
|
||||||
|
|
||||||
|
def test_empty_supported_backend_intersection_keeps_turbowan_choices(self):
|
||||||
|
selected, warning = _resolve_turbo_wan_sparse_backend(
|
||||||
|
attention_type="sla",
|
||||||
|
requested_attention_backend=None,
|
||||||
|
supported_attention_backends={AttentionBackendEnum.FA},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(selected, AttentionBackendEnum.SLA_ATTN)
|
||||||
|
self.assertIsNone(warning)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -42,7 +42,8 @@ DEFAULT_EST_TIME_SECONDS = 300.0
|
|||||||
STARTUP_OVERHEAD_SECONDS = 120.0
|
STARTUP_OVERHEAD_SECONDS = 120.0
|
||||||
|
|
||||||
# Paths relative to repository root
|
# Paths relative to repository root
|
||||||
BASELINE_REL_PATH = "python/sglang/multimodal_gen/test/server/perf_baselines.json"
|
BASELINE_REL_PATH = "python/sglang/multimodal_gen/test/server/perf_baselines"
|
||||||
|
BASELINE_PLATFORM_ORDER = ("h100", "b200", "5090")
|
||||||
RUN_SUITE_REL_PATH = "python/sglang/multimodal_gen/test/run_suite.py"
|
RUN_SUITE_REL_PATH = "python/sglang/multimodal_gen/test/run_suite.py"
|
||||||
|
|
||||||
USE_NPU_CONFIGS = os.getenv("USE_NPU_CONFIGS", "0").lower() in ("1", "true")
|
USE_NPU_CONFIGS = os.getenv("USE_NPU_CONFIGS", "0").lower() in ("1", "true")
|
||||||
@@ -345,28 +346,43 @@ class RunSuiteVisitor(ast.NodeVisitor):
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_baseline_paths(baseline_path: Path) -> List[Path]:
|
||||||
|
if baseline_path.is_file():
|
||||||
|
return [baseline_path]
|
||||||
|
if not baseline_path.is_dir():
|
||||||
|
return []
|
||||||
|
|
||||||
|
ordered_paths = [
|
||||||
|
baseline_path / f"{platform}.json" for platform in BASELINE_PLATFORM_ORDER
|
||||||
|
]
|
||||||
|
ordered_paths.extend(
|
||||||
|
path
|
||||||
|
for path in sorted(baseline_path.glob("*.json"))
|
||||||
|
if path not in ordered_paths
|
||||||
|
)
|
||||||
|
return [path for path in ordered_paths if path.exists()]
|
||||||
|
|
||||||
|
|
||||||
def load_baselines(baseline_path: Path) -> Dict[str, float]:
|
def load_baselines(baseline_path: Path) -> Dict[str, float]:
|
||||||
"""
|
"""
|
||||||
Load performance baselines from JSON file.
|
Load performance baselines from a JSON file or platform baseline directory.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary mapping case_id to estimated time in seconds.
|
Dictionary mapping case_id to estimated time in seconds.
|
||||||
"""
|
"""
|
||||||
if not baseline_path.exists():
|
|
||||||
return {}
|
|
||||||
|
|
||||||
with open(baseline_path, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
baselines = {}
|
baselines = {}
|
||||||
scenarios = data.get("scenarios", {})
|
for path in _iter_baseline_paths(baseline_path):
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
for case_id, scenario in scenarios.items():
|
scenarios = data.get("scenarios", {})
|
||||||
if scenario.get("estimated_full_test_time_s") is not None:
|
for case_id, scenario in scenarios.items():
|
||||||
baselines[case_id] = scenario["estimated_full_test_time_s"]
|
if scenario.get("estimated_full_test_time_s") is not None:
|
||||||
else:
|
est_time = scenario["estimated_full_test_time_s"]
|
||||||
expected_e2e_ms = scenario.get("expected_e2e_ms", 0)
|
else:
|
||||||
baselines[case_id] = expected_e2e_ms / 1000.0 + STARTUP_OVERHEAD_SECONDS
|
expected_e2e_ms = scenario.get("expected_e2e_ms", 0)
|
||||||
|
est_time = expected_e2e_ms / 1000.0 + STARTUP_OVERHEAD_SECONDS
|
||||||
|
baselines.setdefault(case_id, est_time)
|
||||||
|
|
||||||
return baselines
|
return baselines
|
||||||
|
|
||||||
@@ -443,7 +459,7 @@ def collect_diffusion_suites(
|
|||||||
Args:
|
Args:
|
||||||
case_config_path: Path to case config (resolved from run_suite.py)
|
case_config_path: Path to case config (resolved from run_suite.py)
|
||||||
run_suite_path: Path to run_suite.py
|
run_suite_path: Path to run_suite.py
|
||||||
baseline_path: Path to perf_baselines.json
|
baseline_path: Path to perf_baselines/ or a single baseline JSON file
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary mapping suite name to DiffusionSuiteInfo.
|
Dictionary mapping suite name to DiffusionSuiteInfo.
|
||||||
|
|||||||
Reference in New Issue
Block a user