[diffusion] CI: add 5090 job (#29791)

This commit is contained in:
Mick
2026-07-01 19:09:43 +08:00
committed by GitHub
parent 7d9de81cda
commit 79f334b1aa
16 changed files with 731 additions and 85 deletions
@@ -144,6 +144,69 @@ jobs:
with: with:
artifact-suffix: ${{ matrix.part }} artifact-suffix: ${{ matrix.part }}
multimodal-gen-test-1-5090:
if: |
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
inputs.multimodal_gen == 'true'
runs-on: 1-gpu-5090
timeout-minutes: 180
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-pr-test-health
- uses: ./.github/actions/check-maintenance
- name: Download artifacts
if: inputs.sgl_kernel == 'true'
uses: actions/download-artifact@v4
with:
path: sgl-kernel/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda*
- name: Install dependencies
timeout-minutes: 20
run: |
CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
- name: Run 5090 diffusion canary tests
timeout-minutes: 120
env:
RUNAI_STREAMER_MEMORY_LIMIT: 0
CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
run: |
cd python
python3 sglang/multimodal_gen/test/run_suite.py \
--suite 1-gpu-5090 \
$CONTINUE_ON_ERROR_FLAG
- name: Upload execution report
if: always()
uses: actions/upload-artifact@v4
with:
name: diffusion-5090-report-${{ github.run_attempt }}
path: python/sglang/multimodal_gen/test/execution_report_*.json
retention-days: 7
- name: Upload diffusion failure artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: diffusion-failures-5090-${{ github.run_attempt }}
path: diffusion-failures/
if-no-files-found: ignore
retention-days: 7
- uses: ./.github/actions/upload-cuda-coredumps
if: failure()
with:
artifact-suffix: 1-gpu-5090
multimodal-gen-test-2-gpu: multimodal-gen-test-2-gpu:
needs: compute-diffusion-partitions needs: compute-diffusion-partitions
if: | if: |
@@ -42,7 +42,7 @@ def _all_cases() -> list[DiffusionTestCase]:
def _baseline_path() -> Path: def _baseline_path() -> Path:
import sglang.multimodal_gen.test.server.testcase_configs as cfg import sglang.multimodal_gen.test.server.testcase_configs as cfg
return Path(cfg.__file__).with_name("perf_baselines.json") return cfg.get_perf_baseline_path()
def _openai_client(port: int) -> OpenAI: def _openai_client(port: int) -> OpenAI:
@@ -0,0 +1,10 @@
{
"cases": {
"zimage_image_t2i": {
"clip_threshold": 0.92,
"ssim_threshold": 0.86,
"psnr_threshold": 19.5,
"mean_abs_diff_threshold": 8.5
}
}
}
@@ -0,0 +1,3 @@
{
"cases": {}
}
@@ -849,6 +849,80 @@ ONE_GPU_CASES += ONE_GPU_MODELOPT_FP8_CASES
TWO_GPU_CASES = _with_default_num_gpus(TWO_GPU_CASES, 2) TWO_GPU_CASES = _with_default_num_gpus(TWO_GPU_CASES, 2)
ONE_GPU_5090_PERF_CASE_IDS = frozenset(
{
"zimage_image_t2i",
"flux_2_klein_base_image_t2i",
"wan2_1_t2v_1.3b",
}
)
ONE_GPU_5090_SKIP_CONSISTENCY_CASE_IDS = frozenset(
{
"turbo_wan2_1_t2v_1.3b",
}
)
def _select_5090_canary_cases(case_ids: tuple[str, ...]) -> list[DiffusionTestCase]:
cases_by_id = {case.id: case for case in ONE_GPU_CASES}
missing = [case_id for case_id in case_ids if case_id not in cases_by_id]
if missing:
raise RuntimeError(f"Unknown 5090 diffusion canary case(s): {missing}")
return [
replace(
cases_by_id[case_id],
run_perf_check=case_id in ONE_GPU_5090_PERF_CASE_IDS,
run_consistency_check=(
cases_by_id[case_id].run_consistency_check
and case_id not in ONE_GPU_5090_SKIP_CONSISTENCY_CASE_IDS
),
)
for case_id in case_ids
]
def _make_5090_flux_layerwise_cpu_offload_case() -> DiffusionTestCase:
base_case = next(case for case in ONE_GPU_CASES if case.id == "flux_image_t2i")
return replace(
base_case,
id="flux_image_t2i_layerwise_cpu_offload_5090",
server_args=replace(
base_case.server_args,
dit_layerwise_offload=True,
dit_offload_prefetch_size=5,
text_encoder_cpu_offload=True,
extras=[
*base_case.server_args.extras,
"--dit-cpu-offload",
"--pin-cpu-memory",
],
),
sampling_params=replace(
T2I_sampling_params,
output_size="512x512",
extras={"num_inference_steps": 4, "seed": 0},
),
run_perf_check=False,
run_consistency_check=False,
run_component_accuracy_check=False,
run_models_api_check=False,
run_t2v_input_reference_check=False,
)
ONE_GPU_5090_CASES = _select_5090_canary_cases(
(
"zimage_image_t2i",
"flux_2_klein_base_image_t2i",
"wan2_1_t2v_1.3b",
"turbo_wan2_1_t2v_1.3b",
)
)
ONE_GPU_5090_CASES.append(_make_5090_flux_layerwise_cpu_offload_case())
def _discover_unit_tests() -> list[str]: def _discover_unit_tests() -> list[str]:
unit_dir = Path(__file__).resolve().parent.parent / "unit" unit_dir = Path(__file__).resolve().parent.parent / "unit"
if not unit_dir.is_dir(): if not unit_dir.is_dir():
@@ -879,6 +953,9 @@ PARAMETRIZED_CASE_GROUPS = {
"1-gpu": [ "1-gpu": [
("test_server_1_gpu.py", ONE_GPU_CASES), ("test_server_1_gpu.py", ONE_GPU_CASES),
], ],
"1-gpu-5090": [
("test_server_1_gpu_5090.py", ONE_GPU_5090_CASES),
],
"2-gpu": [ "2-gpu": [
("test_server_2_gpu.py", TWO_GPU_CASES), ("test_server_2_gpu.py", TWO_GPU_CASES),
], ],
@@ -0,0 +1,106 @@
{
"metadata": {
"model": "Diffusion Server",
"hardware": "CI RTX 5090 pool",
"description": "Reference numbers captured from real 5090 CI runner history (PR 29791 run 28492141274, job 84450974283).",
"last_updated": "2026-07-01"
},
"tolerances": {
"long_term": {
"e2e": 0.15,
"denoise_stage": 0.1,
"non_denoise_stage": 0.5,
"denoise_step": 0.25,
"denoise_agg": 0.15
},
"pr_test": {
"e2e": 0.25,
"denoise_stage": 0.25,
"non_denoise_stage": 0.8,
"denoise_step": 0.3,
"denoise_agg": 0.2
}
},
"improvement_reporting": {
"threshold": 0.2
},
"sampling": {
"step_fractions": [
0.0,
0.2,
0.4,
0.6,
0.8,
1.0
]
},
"scenarios": {
"flux_2_klein_base_image_t2i": {
"stages_ms": {
"DecodingStage": 18.19,
"DenoisingStage": 18213.48,
"ImageVAEEncodingStage": 0.01,
"InputValidationStage": 0.08,
"LatentPreparationStage": 4.87,
"TextEncodingStage": 104.46,
"TimestepPreparationStage": 142.18
},
"denoise_step_ms": {
"0": 219.58,
"10": 359.79,
"20": 361.55,
"29": 361.99,
"39": 362.57,
"49": 362.89
},
"expected_e2e_ms": 18641.09,
"expected_avg_denoise_ms": 357.93,
"expected_median_denoise_ms": 361.8,
"estimated_full_test_time_s": 94.0
},
"wan2_1_t2v_1.3b": {
"stages_ms": {
"DecodingStage": 1202.43,
"DenoisingStage": 21452.59,
"InputValidationStage": 0.09,
"LatentPreparationStage": 0.23,
"TextEncodingStage": 1210.22,
"TimestepPreparationStage": 3.91
},
"denoise_step_ms": {
"0": 480.39,
"10": 426.11,
"20": 427.32,
"29": 428.09,
"39": 428.04,
"49": 422.31
},
"expected_e2e_ms": 23877.73,
"expected_avg_denoise_ms": 428.74,
"expected_median_denoise_ms": 427.94,
"estimated_full_test_time_s": 160.9
},
"zimage_image_t2i": {
"stages_ms": {
"DecodingStage": 7.11,
"DenoisingStage": 2229.48,
"InputValidationStage": 0.04,
"LatentPreparationStage": 0.17,
"TextEncodingStage": 252.73,
"TimestepPreparationStage": 38.57
},
"denoise_step_ms": {
"0": 39.34,
"2": 277.35,
"3": 267.94,
"5": 279.01,
"6": 273.01,
"8": 275.12
},
"expected_e2e_ms": 2533.95,
"expected_avg_denoise_ms": 246.97,
"expected_median_denoise_ms": 273.01,
"estimated_full_test_time_s": 329.8
}
}
}
@@ -0,0 +1,79 @@
{
"metadata": {
"model": "Diffusion Server",
"hardware": "CI B200 pool",
"description": "Reference estimates for B200-only diffusion cases, split out from the shared diffusion baseline file.",
"last_updated": "2026-07-01"
},
"tolerances": {
"long_term": {
"e2e": 0.15,
"denoise_stage": 0.1,
"non_denoise_stage": 0.5,
"denoise_step": 0.25,
"denoise_agg": 0.15
},
"pr_test": {
"e2e": 0.25,
"denoise_stage": 0.25,
"non_denoise_stage": 0.8,
"denoise_step": 0.3,
"denoise_agg": 0.2
}
},
"improvement_reporting": {
"threshold": 0.2
},
"sampling": {
"step_fractions": [
0.0,
0.2,
0.4,
0.6,
0.8,
1.0
]
},
"scenarios": {
"flux1_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 71.2
},
"flux2_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 592.3
},
"qwen_image_2512_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 120.0
},
"wan22_modelopt_nvfp4_t2v": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 181.8
},
"ideogram4_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 300.0
}
}
}
@@ -2829,38 +2829,6 @@
"expected_avg_denoise_ms": 0.0, "expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0, "expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 73.7 "estimated_full_test_time_s": 73.7
},
"flux1_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 71.2
},
"flux2_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 592.3
},
"qwen_image_2512_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 120.0
},
"wan22_modelopt_nvfp4_t2v": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 181.8
} }
} }
} }
@@ -0,0 +1,20 @@
"""
Config-driven diffusion canary tests for the 1-GPU 5090 PR runner.
"""
from __future__ import annotations
from sglang.multimodal_gen.test.server.common.case_fixtures import (
diffusion_case_fixture,
)
from sglang.multimodal_gen.test.server.gpu_cases import ONE_GPU_5090_CASES
from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401
DiffusionServerBase,
diffusion_server,
)
class TestDiffusionServerOneGpu5090(DiffusionServerBase):
"""Canary tests for lightweight 1-GPU diffusion cases on 5090."""
case = diffusion_case_fixture(ONE_GPU_5090_CASES)
@@ -41,6 +41,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
PerformanceSummary, PerformanceSummary,
ScenarioConfig, ScenarioConfig,
get_model_task_type_for_server_args, get_model_task_type_for_server_args,
get_perf_baseline_path,
) )
from sglang.multimodal_gen.test.test_utils import ( from sglang.multimodal_gen.test.test_utils import (
SGL_TEST_FILES_CI_DATA_REVISION, SGL_TEST_FILES_CI_DATA_REVISION,
@@ -50,6 +51,7 @@ from sglang.multimodal_gen.test.test_utils import (
extract_key_frames_from_video, extract_key_frames_from_video,
get_consistency_gt_candidates, get_consistency_gt_candidates,
get_consistency_gt_remote_files, get_consistency_gt_remote_files,
get_consistency_threshold_path,
get_consistency_thresholds, get_consistency_thresholds,
get_dynamic_server_port, get_dynamic_server_port,
gt_exists, gt_exists,
@@ -232,7 +234,7 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext:
logger.error( logger.error(
f'\n{"=" * 60}\n' f'\n{"=" * 60}\n'
f'Add "estimated_full_test_time_s" to scenario "{case.id}":\n\n' f'Add "estimated_full_test_time_s" to scenario "{case.id}":\n\n'
f"File: python/sglang/multimodal_gen/test/server/perf_baselines.json\n\n" f"File: {get_perf_baseline_path()}\n\n"
f' "{case.id}": {{\n' f' "{case.id}": {{\n'
f" ...\n" f" ...\n"
f' "estimated_full_test_time_s": {_measured_full_time:.1f}\n' f' "estimated_full_test_time_s": {_measured_full_time:.1f}\n'
@@ -441,7 +443,7 @@ class DiffusionServerBase:
self._dump_baseline_for_testcase(case, summary, missing_scenario) self._dump_baseline_for_testcase(case, summary, missing_scenario)
if missing_scenario: if missing_scenario:
pytest.fail( pytest.fail(
f"Testcase '{case.id}' not found in perf_baselines.json" f"Testcase '{case.id}' not found in {get_perf_baseline_path()}"
) )
return return
@@ -552,7 +554,7 @@ class DiffusionServerBase:
) )
action = "add" if missing_scenario else "update" action = "add" if missing_scenario else "update"
output = f""" output = f"""
{action} this baseline in the "scenarios" section of perf_baselines.json: {action} this baseline in the "scenarios" section of {get_perf_baseline_path()}:
"{case.id}": {json.dumps(baseline, indent=4)} "{case.id}": {json.dumps(baseline, indent=4)}
@@ -607,10 +609,10 @@ Add the expected file(s) to sgl-project/ci-data in diffusion-ci/consistency_gt/s
For this case, expected file(s): {names} For this case, expected file(s): {names}
Repository: https://github.com/sgl-project/ci-data (path: diffusion-ci/consistency_gt/sglang_generated/) Repository: https://github.com/sgl-project/ci-data (path: diffusion-ci/consistency_gt/sglang_generated/, with optional platform subdirectories such as 5090/)
Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION} Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
(Optional) Per-case override in consistency_threshold.json: (Optional) Per-case override in {get_consistency_threshold_path()}:
"cases": {{ "cases": {{
"{case.id}": {{ "{case.id}": {{
"clip_threshold": 0.92, "clip_threshold": 0.92,
@@ -12,7 +12,7 @@ pytest python/sglang/multimodal_gen/test/server/test_server_1_gpu.py -k qwen_ima
To add a new testcase: To add a new testcase:
1. add your testcase with case-id: `my_new_test_case_id` to `ONE_GPU_CASES`, `ONE_GPU_MODELOPT_FP8_CASES`, `ONE_GPU_B200_CASES`, or `TWO_GPU_CASES` 1. add your testcase with case-id: `my_new_test_case_id` to `ONE_GPU_CASES`, `ONE_GPU_MODELOPT_FP8_CASES`, `ONE_GPU_B200_CASES`, or `TWO_GPU_CASES`
2. run `SGLANG_GEN_BASELINE=1 pytest -s python/sglang/multimodal_gen/test/server/ -k my_new_test_case_id` 2. run `SGLANG_GEN_BASELINE=1 pytest -s python/sglang/multimodal_gen/test/server/ -k my_new_test_case_id`
3. insert or override the corresponding scenario in `scenarios` section of perf_baselines.json with the output baseline of step-2 3. insert or override the corresponding scenario in the platform JSON under `perf_baselines/`
""" """
@@ -33,6 +33,7 @@ from sglang.multimodal_gen.registry import (
get_model_info, get_model_info,
get_pipeline_config_classes, get_pipeline_config_classes,
) )
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord
@@ -657,6 +658,57 @@ MODELOPT_WAN22_NVFP4_MODEL = "nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4"
MODELOPT_NVFP4_B200_ENV_VARS = {} MODELOPT_NVFP4_B200_ENV_VARS = {}
MODELOPT_WAN22_NVFP4_B200_ENV_VARS = {} MODELOPT_WAN22_NVFP4_B200_ENV_VARS = {}
PERF_BASELINE_PLATFORM_ENV = "SGLANG_DIFFUSION_PERF_BASELINE_PLATFORM"
PERF_BASELINE_DIR = Path(__file__).with_name("perf_baselines")
PERF_BASELINE_FILE_BY_PLATFORM = {
"h100": "h100.json",
"b200": "b200.json",
"5090": "5090.json",
}
PERF_BASELINE_PLATFORM_ALIASES = {
"sm90": "h100",
"hopper": "h100",
"h100": "h100",
"sm100": "b200",
"blackwell": "b200",
"b200": "b200",
"sm120": "5090",
"rtx5090": "5090",
"5090": "5090",
}
def _normalize_perf_baseline_platform(platform: str) -> str:
normalized = platform.strip().lower().replace("_", "-")
normalized = normalized.replace("-", "")
if normalized not in PERF_BASELINE_PLATFORM_ALIASES:
valid = ", ".join(sorted(PERF_BASELINE_FILE_BY_PLATFORM))
raise ValueError(
f"Invalid diffusion perf baseline platform {platform!r}. "
f"Expected one of: {valid}"
)
return PERF_BASELINE_PLATFORM_ALIASES[normalized]
def get_perf_baseline_platform() -> str:
override = os.getenv(PERF_BASELINE_PLATFORM_ENV)
if override:
return _normalize_perf_baseline_platform(override)
if current_platform.is_sm120():
return "5090"
if current_platform.is_blackwell():
return "b200"
return "h100"
def get_perf_baseline_path(platform: str | None = None) -> Path:
baseline_platform = (
_normalize_perf_baseline_platform(platform)
if platform is not None
else get_perf_baseline_platform()
)
return PERF_BASELINE_DIR / PERF_BASELINE_FILE_BY_PLATFORM[baseline_platform]
def _make_modelopt_ci_case( def _make_modelopt_ci_case(
case_id: str, case_id: str,
@@ -694,7 +746,7 @@ def _with_default_num_gpus(
# Load global configuration # Load global configuration
BASELINE_CONFIG = ( BASELINE_CONFIG = (
BaselineConfig.load(Path(__file__).with_name("perf_baselines.json")) BaselineConfig.load(get_perf_baseline_path())
.update(Path(__file__).parent / "ascend" / "perf_baselines_npu.json") .update(Path(__file__).parent / "ascend" / "perf_baselines_npu.json")
.update(Path(__file__).parent / "musa" / "perf_baselines_musa.json") .update(Path(__file__).parent / "musa" / "perf_baselines_musa.json")
) )
+171 -29
View File
@@ -34,7 +34,7 @@ if TYPE_CHECKING:
logger = init_logger(__name__) logger = init_logger(__name__)
SGL_TEST_FILES_CI_DATA_REVISION = "4a271ef34602043f19d253f0d30a5f653fe11325" SGL_TEST_FILES_CI_DATA_REVISION = "702d939e23f17b42183329dace60f221d2587056"
if current_platform.is_npu(): if current_platform.is_npu():
SGL_TEST_FILES_CI_DATA_REVISION = "670d66a8a290b62c0c3c077b3e9b0f4a4d9a44e7" SGL_TEST_FILES_CI_DATA_REVISION = "670d66a8a290b62c0c3c077b3e9b0f4a4d9a44e7"
@@ -79,9 +79,26 @@ SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_CASES = frozenset(
} }
) )
CONSISTENCY_THRESHOLD_JSON_PATH = ( CONSISTENCY_PLATFORM_ENV = "SGLANG_DIFFUSION_CONSISTENCY_PLATFORM"
Path(__file__).resolve().parent / "server" / "consistency_threshold.json" CONSISTENCY_THRESHOLD_DIR = (
Path(__file__).resolve().parent / "server" / "consistency_thresholds"
) )
CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM = {
"h100": "h100.json",
"b200": "b200.json",
"5090": "5090.json",
}
CONSISTENCY_PLATFORM_ALIASES = {
"sm90": "h100",
"hopper": "h100",
"h100": "h100",
"sm100": "b200",
"blackwell": "b200",
"b200": "b200",
"sm120": "5090",
"rtx5090": "5090",
"5090": "5090",
}
CLIP_MODEL_NAME = "openai/clip-vit-large-patch14" CLIP_MODEL_NAME = "openai/clip-vit-large-patch14"
DEFAULT_CLIP_THRESHOLD_IMAGE = 0.92 DEFAULT_CLIP_THRESHOLD_IMAGE = 0.92
DEFAULT_CLIP_THRESHOLD_VIDEO = 0.90 DEFAULT_CLIP_THRESHOLD_VIDEO = 0.90
@@ -694,14 +711,74 @@ def validate_video_file(
), f"Video height mismatch: expected {expected_height}, got {actual_height}" ), f"Video height mismatch: expected {expected_height}, got {actual_height}"
def _load_threshold_json() -> dict[str, Any]: def _normalize_consistency_platform(platform: str) -> str:
"""Load consistency_threshold.json; returns {} if missing.""" normalized = platform.strip().lower().replace("_", "-")
if not CONSISTENCY_THRESHOLD_JSON_PATH.exists(): normalized = normalized.replace("-", "")
if normalized not in CONSISTENCY_PLATFORM_ALIASES:
valid = ", ".join(sorted(CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM))
raise ValueError(
f"Invalid diffusion consistency platform {platform!r}. "
f"Expected one of: {valid}"
)
return CONSISTENCY_PLATFORM_ALIASES[normalized]
def get_consistency_platform() -> str:
override = os.getenv(CONSISTENCY_PLATFORM_ENV)
if override:
return _normalize_consistency_platform(override)
if current_platform.is_sm120():
return "5090"
if current_platform.is_blackwell():
return "b200"
return "h100"
def get_consistency_threshold_path(platform: str | None = None) -> Path:
threshold_platform = (
_normalize_consistency_platform(platform)
if platform is not None
else get_consistency_platform()
)
return (
CONSISTENCY_THRESHOLD_DIR
/ CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM[threshold_platform]
)
def _load_threshold_file(path: Path) -> dict[str, Any]:
if not path.exists():
return {} return {}
with CONSISTENCY_THRESHOLD_JSON_PATH.open("r", encoding="utf-8") as f: with path.open("r", encoding="utf-8") as f:
return json.load(f) return json.load(f)
def _merge_threshold_metadata(
base: dict[str, Any], override: dict[str, Any]
) -> dict[str, Any]:
merged = dict(base)
if "cases" in base or "cases" in override:
merged["cases"] = {
**base.get("cases", {}),
**override.get("cases", {}),
}
for key, value in override.items():
if key != "cases":
merged[key] = value
return merged
def _load_threshold_json() -> dict[str, Any]:
metadata = _load_threshold_file(get_consistency_threshold_path("h100"))
platform = get_consistency_platform()
if platform == "h100":
return metadata
return _merge_threshold_metadata(
metadata,
_load_threshold_file(get_consistency_threshold_path(platform)),
)
@dataclass @dataclass
class ConsistencyThresholds: class ConsistencyThresholds:
clip_threshold: float clip_threshold: float
@@ -958,10 +1035,9 @@ def _consistency_gt_filenames(
return [f"{case_id}_{n}gpu.{ext}"] return [f"{case_id}_{n}gpu.{ext}"]
def get_consistency_gt_candidates( def _base_consistency_gt_candidates(
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
) -> list[str]: ) -> list[str]:
"""Return candidate GT filenames for local consistency data."""
n = num_gpus n = num_gpus
if is_video: if is_video:
return [ return [
@@ -975,6 +1051,31 @@ def get_consistency_gt_candidates(
return [f"{base}.{e}" for e in exts] return [f"{base}.{e}" for e in exts]
def get_consistency_gt_candidate_sets(
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
) -> list[list[str]]:
candidates = _base_consistency_gt_candidates(
case_id, num_gpus, is_video, output_format
)
platform = get_consistency_platform()
if platform == "h100":
return [candidates]
return [[f"{platform}/{candidate}" for candidate in candidates], candidates]
def get_consistency_gt_candidates(
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
) -> list[str]:
"""Return candidate GT filenames for local consistency data."""
return [
candidate
for candidate_set in get_consistency_gt_candidate_sets(
case_id, num_gpus, is_video, output_format
)
for candidate in candidate_set
]
def get_consistency_gt_remote_files( def get_consistency_gt_remote_files(
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
) -> list[tuple[str, str]]: ) -> list[tuple[str, str]]:
@@ -1003,6 +1104,21 @@ def _remote_consistency_gt_candidates(
return [(filename, f"{base_url}/{filename}") for filename in filenames] return [(filename, f"{base_url}/{filename}") for filename in filenames]
def _remote_consistency_gt_candidate_sets(
base_url: str,
case_id: str,
num_gpus: int,
is_video: bool,
output_format: str | None = None,
) -> list[list[tuple[str, str]]]:
return [
[(filename, f"{base_url}/{filename}") for filename in filenames]
for filenames in get_consistency_gt_candidate_sets(
case_id, num_gpus, is_video, output_format
)
]
def _is_ascend_consistency_case(case_id: str) -> bool: def _is_ascend_consistency_case(case_id: str) -> bool:
return "npu" in case_id return "npu" in case_id
@@ -1086,14 +1202,15 @@ def _find_remote_consistency_gt_files(
# Avoid accidentally comparing non-comparable CI cases against official GT. # Avoid accidentally comparing non-comparable CI cases against official GT.
bases = (SGL_TEST_FILES_CONSISTENCY_GT_BASE,) bases = (SGL_TEST_FILES_CONSISTENCY_GT_BASE,)
for base_url in bases: for base_url in bases:
candidates = _remote_consistency_gt_candidates( candidate_sets = _remote_consistency_gt_candidate_sets(
base_url, case_id, num_gpus, is_video, output_format base_url, case_id, num_gpus, is_video, output_format
) )
if is_video: for candidates in candidate_sets:
exists = [_remote_file_exists(url) for _, url in candidates] if is_video:
if all(status is not False for status in exists): exists = [_remote_file_exists(url) for _, url in candidates]
return candidates if all(status is not False for status in exists):
else: return candidates
continue
uncertain_candidate = None uncertain_candidate = None
for filename, url in candidates: for filename, url in candidates:
exists = _remote_file_exists(url) exists = _remote_file_exists(url)
@@ -1122,7 +1239,8 @@ def _get_consistency_gt_cache_key(
) -> str: ) -> str:
gt_dir = _get_consistency_gt_dir() gt_dir = _get_consistency_gt_dir()
source = str(gt_dir) if gt_dir is not None else "remote" source = str(gt_dir) if gt_dir is not None else "remote"
return f"{case_id}:{num_gpus}:{is_video}:{output_format or ''}:{source}" platform = get_consistency_platform()
return f"{platform}:{case_id}:{num_gpus}:{is_video}:{output_format or ''}:{source}"
def load_consistency_gt( def load_consistency_gt(
@@ -1139,29 +1257,43 @@ def load_consistency_gt(
if cached is not None: if cached is not None:
return cached return cached
filenames = _consistency_gt_filenames(case_id, num_gpus, is_video, output_format)
images: list[np.ndarray] = [] images: list[np.ndarray] = []
gt_dir = _get_consistency_gt_dir() gt_dir = _get_consistency_gt_dir()
if gt_dir is not None: if gt_dir is not None:
candidates = get_consistency_gt_candidates( candidate_sets = get_consistency_gt_candidate_sets(
case_id, num_gpus, is_video, output_format case_id, num_gpus, is_video, output_format
) )
if is_video: if is_video:
for fn in candidates: selected = None
path = gt_dir / fn for candidates in candidate_sets:
if not path.exists(): if all((gt_dir / fn).exists() for fn in candidates):
raise FileNotFoundError(f"GT image not found: {path}") selected = candidates
arr = np.array(Image.open(path).convert("RGB")) break
images.append(arr) if selected is None:
tried = ", ".join(
candidate
for candidates in candidate_sets
for candidate in candidates
)
raise FileNotFoundError(
f"GT images not found in {gt_dir}. Tried: {tried}"
)
for fn in selected:
images.append(np.array(Image.open(gt_dir / fn).convert("RGB")))
else: else:
path = None path = None
for fn in candidates: for fn in get_consistency_gt_candidates(
case_id, num_gpus, is_video, output_format
):
candidate = gt_dir / fn candidate = gt_dir / fn
if candidate.exists(): if candidate.exists():
path = candidate path = candidate
break break
if path is None: if path is None:
candidates = get_consistency_gt_candidates(
case_id, num_gpus, is_video, output_format
)
raise FileNotFoundError( raise FileNotFoundError(
f"GT image not found in {gt_dir}. Tried: {', '.join(candidates)}" f"GT image not found in {gt_dir}. Tried: {', '.join(candidates)}"
) )
@@ -1172,8 +1304,11 @@ def load_consistency_gt(
case_id, num_gpus, is_video, output_format case_id, num_gpus, is_video, output_format
) )
if not remote_files: if not remote_files:
candidates = get_consistency_gt_candidates(
case_id, num_gpus, is_video, output_format
)
raise FileNotFoundError( raise FileNotFoundError(
f"GT image not found for {case_id}. Tried: {', '.join(filenames)}" f"GT image not found for {case_id}. Tried: {', '.join(candidates)}"
) )
for _, url in remote_files: for _, url in remote_files:
images.append(_load_remote_gt_image(url)) images.append(_load_remote_gt_image(url))
@@ -1210,12 +1345,19 @@ def gt_exists(
"""Check whether GT image(s) exist.""" """Check whether GT image(s) exist."""
gt_dir = _get_consistency_gt_dir() gt_dir = _get_consistency_gt_dir()
if gt_dir is not None: if gt_dir is not None:
candidates = get_consistency_gt_candidates( candidate_sets = get_consistency_gt_candidate_sets(
case_id, num_gpus, is_video, output_format case_id, num_gpus, is_video, output_format
) )
if is_video: if is_video:
return all((gt_dir / c).exists() for c in candidates) return any(
return any((gt_dir / c).exists() for c in candidates) all((gt_dir / candidate).exists() for candidate in candidate_set)
for candidate_set in candidate_sets
)
return any(
(gt_dir / candidate).exists()
for candidate_set in candidate_sets
for candidate in candidate_set
)
cache_key = _get_consistency_gt_cache_key( cache_key = _get_consistency_gt_cache_key(
case_id, num_gpus, is_video, output_format case_id, num_gpus, is_video, output_format
@@ -54,6 +54,53 @@ def test_remote_video_gt_candidates_survive_inconclusive_probe(monkeypatch):
] ]
def test_platform_gt_candidates_prefer_platform_then_default(monkeypatch):
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "5090")
assert test_utils.get_consistency_gt_candidates(
"unit_image",
1,
is_video=False,
output_format="png",
) == [
"5090/unit_image_1gpu.png",
"5090/unit_image_1gpu.jpg",
"5090/unit_image_1gpu.webp",
"unit_image_1gpu.png",
"unit_image_1gpu.jpg",
"unit_image_1gpu.webp",
]
def test_threshold_metadata_merges_platform_override():
metadata = test_utils._merge_threshold_metadata(
{
"cases": {
"case_a": {
"clip_threshold": 0.9,
"ssim_threshold": 0.9,
"psnr_threshold": 20.0,
"mean_abs_diff_threshold": 10.0,
}
},
"default_clip_threshold_image": 0.92,
},
{
"cases": {
"case_a": {
"clip_threshold": 0.8,
"ssim_threshold": 0.7,
"psnr_threshold": 12.0,
"mean_abs_diff_threshold": 20.0,
}
}
},
)
assert metadata["default_clip_threshold_image"] == 0.92
assert metadata["cases"]["case_a"]["psnr_threshold"] == 12.0
def test_pixel_metrics_identical_image(): def test_pixel_metrics_identical_image():
image = _solid_image(128) image = _solid_image(128)
@@ -0,0 +1,61 @@
import unittest
from sglang.multimodal_gen.runtime.layers.attention.turbo_layer import (
_resolve_turbo_wan_sparse_backend,
)
from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum
class TestTurboWanBackendSelection(unittest.TestCase):
def test_non_sparse_requested_backend_falls_back_to_attention_type(self):
selected, warning = _resolve_turbo_wan_sparse_backend(
attention_type="sla",
requested_attention_backend="fa",
)
self.assertEqual(selected, AttentionBackendEnum.SLA_ATTN)
self.assertIsNotNone(warning)
self.assertIn("TurboWan only supports", warning)
self.assertIn("attention_backend='fa'", warning)
def test_sagesla_attention_type_prefers_sage_sparse_backend(self):
selected, warning = _resolve_turbo_wan_sparse_backend(
attention_type="sagesla",
requested_attention_backend="torch_sdpa",
)
self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN)
self.assertIsNotNone(warning)
def test_requested_sparse_backend_is_honored(self):
selected, warning = _resolve_turbo_wan_sparse_backend(
attention_type="sla",
requested_attention_backend="sage_sla_attn",
)
self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN)
self.assertIsNone(warning)
def test_supported_backend_filter_is_respected(self):
selected, warning = _resolve_turbo_wan_sparse_backend(
attention_type="sla",
requested_attention_backend=None,
supported_attention_backends={AttentionBackendEnum.SAGE_SLA_ATTN},
)
self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN)
self.assertIsNone(warning)
def test_empty_supported_backend_intersection_keeps_turbowan_choices(self):
selected, warning = _resolve_turbo_wan_sparse_backend(
attention_type="sla",
requested_attention_backend=None,
supported_attention_backends={AttentionBackendEnum.FA},
)
self.assertEqual(selected, AttentionBackendEnum.SLA_ATTN)
self.assertIsNone(warning)
if __name__ == "__main__":
unittest.main()
@@ -42,7 +42,8 @@ DEFAULT_EST_TIME_SECONDS = 300.0
STARTUP_OVERHEAD_SECONDS = 120.0 STARTUP_OVERHEAD_SECONDS = 120.0
# Paths relative to repository root # Paths relative to repository root
BASELINE_REL_PATH = "python/sglang/multimodal_gen/test/server/perf_baselines.json" BASELINE_REL_PATH = "python/sglang/multimodal_gen/test/server/perf_baselines"
BASELINE_PLATFORM_ORDER = ("h100", "b200", "5090")
RUN_SUITE_REL_PATH = "python/sglang/multimodal_gen/test/run_suite.py" RUN_SUITE_REL_PATH = "python/sglang/multimodal_gen/test/run_suite.py"
USE_NPU_CONFIGS = os.getenv("USE_NPU_CONFIGS", "0").lower() in ("1", "true") USE_NPU_CONFIGS = os.getenv("USE_NPU_CONFIGS", "0").lower() in ("1", "true")
@@ -345,28 +346,43 @@ class RunSuiteVisitor(ast.NodeVisitor):
return result return result
def _iter_baseline_paths(baseline_path: Path) -> List[Path]:
if baseline_path.is_file():
return [baseline_path]
if not baseline_path.is_dir():
return []
ordered_paths = [
baseline_path / f"{platform}.json" for platform in BASELINE_PLATFORM_ORDER
]
ordered_paths.extend(
path
for path in sorted(baseline_path.glob("*.json"))
if path not in ordered_paths
)
return [path for path in ordered_paths if path.exists()]
def load_baselines(baseline_path: Path) -> Dict[str, float]: def load_baselines(baseline_path: Path) -> Dict[str, float]:
""" """
Load performance baselines from JSON file. Load performance baselines from a JSON file or platform baseline directory.
Returns: Returns:
Dictionary mapping case_id to estimated time in seconds. Dictionary mapping case_id to estimated time in seconds.
""" """
if not baseline_path.exists():
return {}
with open(baseline_path, "r", encoding="utf-8") as f:
data = json.load(f)
baselines = {} baselines = {}
scenarios = data.get("scenarios", {}) for path in _iter_baseline_paths(baseline_path):
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
for case_id, scenario in scenarios.items(): scenarios = data.get("scenarios", {})
if scenario.get("estimated_full_test_time_s") is not None: for case_id, scenario in scenarios.items():
baselines[case_id] = scenario["estimated_full_test_time_s"] if scenario.get("estimated_full_test_time_s") is not None:
else: est_time = scenario["estimated_full_test_time_s"]
expected_e2e_ms = scenario.get("expected_e2e_ms", 0) else:
baselines[case_id] = expected_e2e_ms / 1000.0 + STARTUP_OVERHEAD_SECONDS expected_e2e_ms = scenario.get("expected_e2e_ms", 0)
est_time = expected_e2e_ms / 1000.0 + STARTUP_OVERHEAD_SECONDS
baselines.setdefault(case_id, est_time)
return baselines return baselines
@@ -443,7 +459,7 @@ def collect_diffusion_suites(
Args: Args:
case_config_path: Path to case config (resolved from run_suite.py) case_config_path: Path to case config (resolved from run_suite.py)
run_suite_path: Path to run_suite.py run_suite_path: Path to run_suite.py
baseline_path: Path to perf_baselines.json baseline_path: Path to perf_baselines/ or a single baseline JSON file
Returns: Returns:
Dictionary mapping suite name to DiffusionSuiteInfo. Dictionary mapping suite name to DiffusionSuiteInfo.