minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[diffusion] CI: add 5090 job (#29791)

Browse Source
This commit is contained in:
Mick
2026-07-01 19:09:43 +08:00
committed by GitHub
parent 7d9de81cda
commit 79f334b1aa
16 changed files with 731 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py
+1 -1
View File
@@ -42,7 +42,7 @@ def _all_cases() -> list[DiffusionTestCase]:
def _baseline_path() -> Path:
import sglang.multimodal_gen.test.server.testcase_configs as cfg
return Path(cfg.__file__).with_name("perf_baselines.json")
return cfg.get_perf_baseline_path()
def _openai_client(port: int) -> OpenAI:
python/sglang/multimodal_gen/test/server/consistency_thresholds/5090.json
+10
View File
@@ -0,0 +1,10 @@
{
"cases": {
"zimage_image_t2i": {
"clip_threshold": 0.92,
"ssim_threshold": 0.86,
"psnr_threshold": 19.5,
"mean_abs_diff_threshold": 8.5
}
}
}
python/sglang/multimodal_gen/test/server/consistency_thresholds/b200.json
+3
View File
@@ -0,0 +1,3 @@
{
"cases": {}
}
python/sglang/multimodal_gen/test/server/consistency_threshold.json → python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json
View File
python/sglang/multimodal_gen/test/server/gpu_cases.py
+77
View File
@@ -849,6 +849,80 @@ ONE_GPU_CASES += ONE_GPU_MODELOPT_FP8_CASES
TWO_GPU_CASES = _with_default_num_gpus(TWO_GPU_CASES, 2)
ONE_GPU_5090_PERF_CASE_IDS = frozenset(
{
"zimage_image_t2i",
"flux_2_klein_base_image_t2i",
"wan2_1_t2v_1.3b",
}
)
ONE_GPU_5090_SKIP_CONSISTENCY_CASE_IDS = frozenset(
{
"turbo_wan2_1_t2v_1.3b",
}
)
def _select_5090_canary_cases(case_ids: tuple[str, ...]) -> list[DiffusionTestCase]:
cases_by_id = {case.id: case for case in ONE_GPU_CASES}
missing = [case_id for case_id in case_ids if case_id not in cases_by_id]
if missing:
raise RuntimeError(f"Unknown 5090 diffusion canary case(s): {missing}")
return [
replace(
cases_by_id[case_id],
run_perf_check=case_id in ONE_GPU_5090_PERF_CASE_IDS,
run_consistency_check=(
cases_by_id[case_id].run_consistency_check
and case_id not in ONE_GPU_5090_SKIP_CONSISTENCY_CASE_IDS
),
)
for case_id in case_ids
]
def _make_5090_flux_layerwise_cpu_offload_case() -> DiffusionTestCase:
base_case = next(case for case in ONE_GPU_CASES if case.id == "flux_image_t2i")
return replace(
base_case,
id="flux_image_t2i_layerwise_cpu_offload_5090",
server_args=replace(
base_case.server_args,
dit_layerwise_offload=True,
dit_offload_prefetch_size=5,
text_encoder_cpu_offload=True,
extras=[
*base_case.server_args.extras,
"--dit-cpu-offload",
"--pin-cpu-memory",
],
),
sampling_params=replace(
T2I_sampling_params,
output_size="512x512",
extras={"num_inference_steps": 4, "seed": 0},
),
run_perf_check=False,
run_consistency_check=False,
run_component_accuracy_check=False,
run_models_api_check=False,
run_t2v_input_reference_check=False,
)
ONE_GPU_5090_CASES = _select_5090_canary_cases(
(
"zimage_image_t2i",
"flux_2_klein_base_image_t2i",
"wan2_1_t2v_1.3b",
"turbo_wan2_1_t2v_1.3b",
)
)
ONE_GPU_5090_CASES.append(_make_5090_flux_layerwise_cpu_offload_case())
def _discover_unit_tests() -> list[str]:
unit_dir = Path(__file__).resolve().parent.parent / "unit"
if not unit_dir.is_dir():
@@ -879,6 +953,9 @@ PARAMETRIZED_CASE_GROUPS = {
"1-gpu": [
("test_server_1_gpu.py", ONE_GPU_CASES),
],
"1-gpu-5090": [
("test_server_1_gpu_5090.py", ONE_GPU_5090_CASES),
],
"2-gpu": [
("test_server_2_gpu.py", TWO_GPU_CASES),
],
python/sglang/multimodal_gen/test/server/perf_baselines/5090.json
+106
View File
@@ -0,0 +1,106 @@
{
"metadata": {
"model": "Diffusion Server",
"hardware": "CI RTX 5090 pool",
"description": "Reference numbers captured from real 5090 CI runner history (PR 29791 run 28492141274, job 84450974283).",
"last_updated": "2026-07-01"
},
"tolerances": {
"long_term": {
"e2e": 0.15,
"denoise_stage": 0.1,
"non_denoise_stage": 0.5,
"denoise_step": 0.25,
"denoise_agg": 0.15
},
"pr_test": {
"e2e": 0.25,
"denoise_stage": 0.25,
"non_denoise_stage": 0.8,
"denoise_step": 0.3,
"denoise_agg": 0.2
}
},
"improvement_reporting": {
"threshold": 0.2
},
"sampling": {
"step_fractions": [
0.0,
0.2,
0.4,
0.6,
0.8,
1.0
]
},
"scenarios": {
"flux_2_klein_base_image_t2i": {
"stages_ms": {
"DecodingStage": 18.19,
"DenoisingStage": 18213.48,
"ImageVAEEncodingStage": 0.01,
"InputValidationStage": 0.08,
"LatentPreparationStage": 4.87,
"TextEncodingStage": 104.46,
"TimestepPreparationStage": 142.18
},
"denoise_step_ms": {
"0": 219.58,
"10": 359.79,
"20": 361.55,
"29": 361.99,
"39": 362.57,
"49": 362.89
},
"expected_e2e_ms": 18641.09,
"expected_avg_denoise_ms": 357.93,
"expected_median_denoise_ms": 361.8,
"estimated_full_test_time_s": 94.0
},
"wan2_1_t2v_1.3b": {
"stages_ms": {
"DecodingStage": 1202.43,
"DenoisingStage": 21452.59,
"InputValidationStage": 0.09,
"LatentPreparationStage": 0.23,
"TextEncodingStage": 1210.22,
"TimestepPreparationStage": 3.91
},
"denoise_step_ms": {
"0": 480.39,
"10": 426.11,
"20": 427.32,
"29": 428.09,
"39": 428.04,
"49": 422.31
},
"expected_e2e_ms": 23877.73,
"expected_avg_denoise_ms": 428.74,
"expected_median_denoise_ms": 427.94,
"estimated_full_test_time_s": 160.9
},
"zimage_image_t2i": {
"stages_ms": {
"DecodingStage": 7.11,
"DenoisingStage": 2229.48,
"InputValidationStage": 0.04,
"LatentPreparationStage": 0.17,
"TextEncodingStage": 252.73,
"TimestepPreparationStage": 38.57
},
"denoise_step_ms": {
"0": 39.34,
"2": 277.35,
"3": 267.94,
"5": 279.01,
"6": 273.01,
"8": 275.12
},
"expected_e2e_ms": 2533.95,
"expected_avg_denoise_ms": 246.97,
"expected_median_denoise_ms": 273.01,
"estimated_full_test_time_s": 329.8
}
}
}
python/sglang/multimodal_gen/test/server/perf_baselines/b200.json
+79
View File
@@ -0,0 +1,79 @@
{
"metadata": {
"model": "Diffusion Server",
"hardware": "CI B200 pool",
"description": "Reference estimates for B200-only diffusion cases, split out from the shared diffusion baseline file.",
"last_updated": "2026-07-01"
},
"tolerances": {
"long_term": {
"e2e": 0.15,
"denoise_stage": 0.1,
"non_denoise_stage": 0.5,
"denoise_step": 0.25,
"denoise_agg": 0.15
},
"pr_test": {
"e2e": 0.25,
"denoise_stage": 0.25,
"non_denoise_stage": 0.8,
"denoise_step": 0.3,
"denoise_agg": 0.2
}
},
"improvement_reporting": {
"threshold": 0.2
},
"sampling": {
"step_fractions": [
0.0,
0.2,
0.4,
0.6,
0.8,
1.0
]
},
"scenarios": {
"flux1_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 71.2
},
"flux2_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 592.3
},
"qwen_image_2512_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 120.0
},
"wan22_modelopt_nvfp4_t2v": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 181.8
},
"ideogram4_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 300.0
}
}
}
python/sglang/multimodal_gen/test/server/perf_baselines.json → python/sglang/multimodal_gen/test/server/perf_baselines/h100.json
-32
View File
@@ -2829,38 +2829,6 @@
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 73.7
},
"flux1_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 71.2
},
"flux2_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 592.3
},
"qwen_image_2512_modelopt_nvfp4_t2i": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 120.0
},
"wan22_modelopt_nvfp4_t2v": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 181.8
}
}
}
python/sglang/multimodal_gen/test/server/test_server_1_gpu_5090.py
+20
View File
@@ -0,0 +1,20 @@
"""
Config-driven diffusion canary tests for the 1-GPU 5090 PR runner.
"""
from __future__ import annotations
from sglang.multimodal_gen.test.server.common.case_fixtures import (
diffusion_case_fixture,
)
from sglang.multimodal_gen.test.server.gpu_cases import ONE_GPU_5090_CASES
from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401
DiffusionServerBase,
diffusion_server,
)
class TestDiffusionServerOneGpu5090(DiffusionServerBase):
"""Canary tests for lightweight 1-GPU diffusion cases on 5090."""
case = diffusion_case_fixture(ONE_GPU_5090_CASES)
python/sglang/multimodal_gen/test/server/test_server_common.py
+7 -5
View File
@@ -41,6 +41,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
PerformanceSummary,
ScenarioConfig,
get_model_task_type_for_server_args,
get_perf_baseline_path,
)
from sglang.multimodal_gen.test.test_utils import (
SGL_TEST_FILES_CI_DATA_REVISION,
@@ -50,6 +51,7 @@ from sglang.multimodal_gen.test.test_utils import (
extract_key_frames_from_video,
get_consistency_gt_candidates,
get_consistency_gt_remote_files,
get_consistency_threshold_path,
get_consistency_thresholds,
get_dynamic_server_port,
gt_exists,
@@ -232,7 +234,7 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext:
logger.error(
f'\n{"=" * 60}\n'
f'Add "estimated_full_test_time_s" to scenario "{case.id}":\n\n'
f"File: python/sglang/multimodal_gen/test/server/perf_baselines.json\n\n"
f"File: {get_perf_baseline_path()}\n\n"
f' "{case.id}": {{\n'
f" ...\n"
f' "estimated_full_test_time_s": {_measured_full_time:.1f}\n'
@@ -441,7 +443,7 @@ class DiffusionServerBase:
self._dump_baseline_for_testcase(case, summary, missing_scenario)
if missing_scenario:
pytest.fail(
f"Testcase '{case.id}' not found in perf_baselines.json"
f"Testcase '{case.id}' not found in {get_perf_baseline_path()}"
)
return
@@ -552,7 +554,7 @@ class DiffusionServerBase:
)
action = "add" if missing_scenario else "update"
output = f"""
{action} this baseline in the "scenarios" section of perf_baselines.json:
{action} this baseline in the "scenarios" section of {get_perf_baseline_path()}:
"{case.id}": {json.dumps(baseline, indent=4)}
@@ -607,10 +609,10 @@ Add the expected file(s) to sgl-project/ci-data in diffusion-ci/consistency_gt/s
For this case, expected file(s): {names}
Repository: https://github.com/sgl-project/ci-data (path: diffusion-ci/consistency_gt/sglang_generated/)
Repository: https://github.com/sgl-project/ci-data (path: diffusion-ci/consistency_gt/sglang_generated/, with optional platform subdirectories such as 5090/)
Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
(Optional) Per-case override in consistency_threshold.json:
(Optional) Per-case override in {get_consistency_threshold_path()}:
"cases": {{
"{case.id}": {{
"clip_threshold": 0.92,
python/sglang/multimodal_gen/test/server/testcase_configs.py
+54 -2
View File
@@ -12,7 +12,7 @@ pytest python/sglang/multimodal_gen/test/server/test_server_1_gpu.py -k qwen_ima
To add a new testcase:
1. add your testcase with case-id: `my_new_test_case_id` to `ONE_GPU_CASES`, `ONE_GPU_MODELOPT_FP8_CASES`, `ONE_GPU_B200_CASES`, or `TWO_GPU_CASES`
2. run `SGLANG_GEN_BASELINE=1 pytest -s python/sglang/multimodal_gen/test/server/ -k my_new_test_case_id`
3. insert or override the corresponding scenario in `scenarios` section of perf_baselines.json with the output baseline of step-2
3. insert or override the corresponding scenario in the platform JSON under `perf_baselines/`
"""
@@ -33,6 +33,7 @@ from sglang.multimodal_gen.registry import (
get_model_info,
get_pipeline_config_classes,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord
@@ -657,6 +658,57 @@ MODELOPT_WAN22_NVFP4_MODEL = "nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4"
MODELOPT_NVFP4_B200_ENV_VARS = {}
MODELOPT_WAN22_NVFP4_B200_ENV_VARS = {}
PERF_BASELINE_PLATFORM_ENV = "SGLANG_DIFFUSION_PERF_BASELINE_PLATFORM"
PERF_BASELINE_DIR = Path(__file__).with_name("perf_baselines")
PERF_BASELINE_FILE_BY_PLATFORM = {
"h100": "h100.json",
"b200": "b200.json",
"5090": "5090.json",
}
PERF_BASELINE_PLATFORM_ALIASES = {
"sm90": "h100",
"hopper": "h100",
"h100": "h100",
"sm100": "b200",
"blackwell": "b200",
"b200": "b200",
"sm120": "5090",
"rtx5090": "5090",
"5090": "5090",
}
def _normalize_perf_baseline_platform(platform: str) -> str:
normalized = platform.strip().lower().replace("_", "-")
normalized = normalized.replace("-", "")
if normalized not in PERF_BASELINE_PLATFORM_ALIASES:
valid = ", ".join(sorted(PERF_BASELINE_FILE_BY_PLATFORM))
raise ValueError(
f"Invalid diffusion perf baseline platform {platform!r}. "
f"Expected one of: {valid}"
)
return PERF_BASELINE_PLATFORM_ALIASES[normalized]
def get_perf_baseline_platform() -> str:
override = os.getenv(PERF_BASELINE_PLATFORM_ENV)
if override:
return _normalize_perf_baseline_platform(override)
if current_platform.is_sm120():
return "5090"
if current_platform.is_blackwell():
return "b200"
return "h100"
def get_perf_baseline_path(platform: str | None = None) -> Path:
baseline_platform = (
_normalize_perf_baseline_platform(platform)
if platform is not None
else get_perf_baseline_platform()
)
return PERF_BASELINE_DIR / PERF_BASELINE_FILE_BY_PLATFORM[baseline_platform]
def _make_modelopt_ci_case(
case_id: str,
@@ -694,7 +746,7 @@ def _with_default_num_gpus(
# Load global configuration
BASELINE_CONFIG = (
BaselineConfig.load(Path(__file__).with_name("perf_baselines.json"))
BaselineConfig.load(get_perf_baseline_path())
.update(Path(__file__).parent / "ascend" / "perf_baselines_npu.json")
.update(Path(__file__).parent / "musa" / "perf_baselines_musa.json")
)
python/sglang/multimodal_gen/test/test_utils.py
+171 -29
View File
@@ -34,7 +34,7 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
SGL_TEST_FILES_CI_DATA_REVISION = "4a271ef34602043f19d253f0d30a5f653fe11325"
SGL_TEST_FILES_CI_DATA_REVISION = "702d939e23f17b42183329dace60f221d2587056"
if current_platform.is_npu():
SGL_TEST_FILES_CI_DATA_REVISION = "670d66a8a290b62c0c3c077b3e9b0f4a4d9a44e7"
@@ -79,9 +79,26 @@ SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_CASES = frozenset(
}
)
CONSISTENCY_THRESHOLD_JSON_PATH = (
Path(__file__).resolve().parent / "server" / "consistency_threshold.json"
CONSISTENCY_PLATFORM_ENV = "SGLANG_DIFFUSION_CONSISTENCY_PLATFORM"
CONSISTENCY_THRESHOLD_DIR = (
Path(__file__).resolve().parent / "server" / "consistency_thresholds"
)
CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM = {
"h100": "h100.json",
"b200": "b200.json",
"5090": "5090.json",
}
CONSISTENCY_PLATFORM_ALIASES = {
"sm90": "h100",
"hopper": "h100",
"h100": "h100",
"sm100": "b200",
"blackwell": "b200",
"b200": "b200",
"sm120": "5090",
"rtx5090": "5090",
"5090": "5090",
}
CLIP_MODEL_NAME = "openai/clip-vit-large-patch14"
DEFAULT_CLIP_THRESHOLD_IMAGE = 0.92
DEFAULT_CLIP_THRESHOLD_VIDEO = 0.90
@@ -694,14 +711,74 @@ def validate_video_file(
), f"Video height mismatch: expected {expected_height}, got {actual_height}"
def _load_threshold_json() -> dict[str, Any]:
"""Load consistency_threshold.json; returns {} if missing."""
if not CONSISTENCY_THRESHOLD_JSON_PATH.exists():
def _normalize_consistency_platform(platform: str) -> str:
normalized = platform.strip().lower().replace("_", "-")
normalized = normalized.replace("-", "")
if normalized not in CONSISTENCY_PLATFORM_ALIASES:
valid = ", ".join(sorted(CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM))
raise ValueError(
f"Invalid diffusion consistency platform {platform!r}. "
f"Expected one of: {valid}"
)
return CONSISTENCY_PLATFORM_ALIASES[normalized]
def get_consistency_platform() -> str:
override = os.getenv(CONSISTENCY_PLATFORM_ENV)
if override:
return _normalize_consistency_platform(override)
if current_platform.is_sm120():
return "5090"
if current_platform.is_blackwell():
return "b200"
return "h100"
def get_consistency_threshold_path(platform: str | None = None) -> Path:
threshold_platform = (
_normalize_consistency_platform(platform)
if platform is not None
else get_consistency_platform()
)
return (
CONSISTENCY_THRESHOLD_DIR
/ CONSISTENCY_THRESHOLD_FILE_BY_PLATFORM[threshold_platform]
)
def _load_threshold_file(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
with CONSISTENCY_THRESHOLD_JSON_PATH.open("r", encoding="utf-8") as f:
with path.open("r", encoding="utf-8") as f:
return json.load(f)
def _merge_threshold_metadata(
base: dict[str, Any], override: dict[str, Any]
) -> dict[str, Any]:
merged = dict(base)
if "cases" in base or "cases" in override:
merged["cases"] = {
**base.get("cases", {}),
**override.get("cases", {}),
}
for key, value in override.items():
if key != "cases":
merged[key] = value
return merged
def _load_threshold_json() -> dict[str, Any]:
metadata = _load_threshold_file(get_consistency_threshold_path("h100"))
platform = get_consistency_platform()
if platform == "h100":
return metadata
return _merge_threshold_metadata(
metadata,
_load_threshold_file(get_consistency_threshold_path(platform)),
)
@dataclass
class ConsistencyThresholds:
clip_threshold: float
@@ -958,10 +1035,9 @@ def _consistency_gt_filenames(
return [f"{case_id}_{n}gpu.{ext}"]
def get_consistency_gt_candidates(
def _base_consistency_gt_candidates(
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
) -> list[str]:
"""Return candidate GT filenames for local consistency data."""
n = num_gpus
if is_video:
return [
@@ -975,6 +1051,31 @@ def get_consistency_gt_candidates(
return [f"{base}.{e}" for e in exts]
def get_consistency_gt_candidate_sets(
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
) -> list[list[str]]:
candidates = _base_consistency_gt_candidates(
case_id, num_gpus, is_video, output_format
)
platform = get_consistency_platform()
if platform == "h100":
return [candidates]
return [[f"{platform}/{candidate}" for candidate in candidates], candidates]
def get_consistency_gt_candidates(
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
) -> list[str]:
"""Return candidate GT filenames for local consistency data."""
return [
candidate
for candidate_set in get_consistency_gt_candidate_sets(
case_id, num_gpus, is_video, output_format
)
for candidate in candidate_set
]
def get_consistency_gt_remote_files(
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
) -> list[tuple[str, str]]:
@@ -1003,6 +1104,21 @@ def _remote_consistency_gt_candidates(
return [(filename, f"{base_url}/{filename}") for filename in filenames]
def _remote_consistency_gt_candidate_sets(
base_url: str,
case_id: str,
num_gpus: int,
is_video: bool,
output_format: str | None = None,
) -> list[list[tuple[str, str]]]:
return [
[(filename, f"{base_url}/{filename}") for filename in filenames]
for filenames in get_consistency_gt_candidate_sets(
case_id, num_gpus, is_video, output_format
)
]
def _is_ascend_consistency_case(case_id: str) -> bool:
return "npu" in case_id
@@ -1086,14 +1202,15 @@ def _find_remote_consistency_gt_files(
# Avoid accidentally comparing non-comparable CI cases against official GT.
bases = (SGL_TEST_FILES_CONSISTENCY_GT_BASE,)
for base_url in bases:
candidates = _remote_consistency_gt_candidates(
candidate_sets = _remote_consistency_gt_candidate_sets(
base_url, case_id, num_gpus, is_video, output_format
)
if is_video:
exists = [_remote_file_exists(url) for _, url in candidates]
if all(status is not False for status in exists):
return candidates
else:
for candidates in candidate_sets:
if is_video:
exists = [_remote_file_exists(url) for _, url in candidates]
if all(status is not False for status in exists):
return candidates
continue
uncertain_candidate = None
for filename, url in candidates:
exists = _remote_file_exists(url)
@@ -1122,7 +1239,8 @@ def _get_consistency_gt_cache_key(
) -> str:
gt_dir = _get_consistency_gt_dir()
source = str(gt_dir) if gt_dir is not None else "remote"
return f"{case_id}:{num_gpus}:{is_video}:{output_format or ''}:{source}"
platform = get_consistency_platform()
return f"{platform}:{case_id}:{num_gpus}:{is_video}:{output_format or ''}:{source}"
def load_consistency_gt(
@@ -1139,29 +1257,43 @@ def load_consistency_gt(
if cached is not None:
return cached
filenames = _consistency_gt_filenames(case_id, num_gpus, is_video, output_format)
images: list[np.ndarray] = []
gt_dir = _get_consistency_gt_dir()
if gt_dir is not None:
candidates = get_consistency_gt_candidates(
candidate_sets = get_consistency_gt_candidate_sets(
case_id, num_gpus, is_video, output_format
)
if is_video:
for fn in candidates:
path = gt_dir / fn
if not path.exists():
raise FileNotFoundError(f"GT image not found: {path}")
arr = np.array(Image.open(path).convert("RGB"))
images.append(arr)
selected = None
for candidates in candidate_sets:
if all((gt_dir / fn).exists() for fn in candidates):
selected = candidates
break
if selected is None:
tried = ", ".join(
candidate
for candidates in candidate_sets
for candidate in candidates
)
raise FileNotFoundError(
f"GT images not found in {gt_dir}. Tried: {tried}"
)
for fn in selected:
images.append(np.array(Image.open(gt_dir / fn).convert("RGB")))
else:
path = None
for fn in candidates:
for fn in get_consistency_gt_candidates(
case_id, num_gpus, is_video, output_format
):
candidate = gt_dir / fn
if candidate.exists():
path = candidate
break
if path is None:
candidates = get_consistency_gt_candidates(
case_id, num_gpus, is_video, output_format
)
raise FileNotFoundError(
f"GT image not found in {gt_dir}. Tried: {', '.join(candidates)}"
)
@@ -1172,8 +1304,11 @@ def load_consistency_gt(
case_id, num_gpus, is_video, output_format
)
if not remote_files:
candidates = get_consistency_gt_candidates(
case_id, num_gpus, is_video, output_format
)
raise FileNotFoundError(
f"GT image not found for {case_id}. Tried: {', '.join(filenames)}"
f"GT image not found for {case_id}. Tried: {', '.join(candidates)}"
)
for _, url in remote_files:
images.append(_load_remote_gt_image(url))
@@ -1210,12 +1345,19 @@ def gt_exists(
"""Check whether GT image(s) exist."""
gt_dir = _get_consistency_gt_dir()
if gt_dir is not None:
candidates = get_consistency_gt_candidates(
candidate_sets = get_consistency_gt_candidate_sets(
case_id, num_gpus, is_video, output_format
)
if is_video:
return all((gt_dir / c).exists() for c in candidates)
return any((gt_dir / c).exists() for c in candidates)
return any(
all((gt_dir / candidate).exists() for candidate in candidate_set)
for candidate_set in candidate_sets
)
return any(
(gt_dir / candidate).exists()
for candidate_set in candidate_sets
for candidate in candidate_set
)
cache_key = _get_consistency_gt_cache_key(
case_id, num_gpus, is_video, output_format
python/sglang/multimodal_gen/test/unit/test_consistency_metrics.py
+47
View File
@@ -54,6 +54,53 @@ def test_remote_video_gt_candidates_survive_inconclusive_probe(monkeypatch):
]
def test_platform_gt_candidates_prefer_platform_then_default(monkeypatch):
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "5090")
assert test_utils.get_consistency_gt_candidates(
"unit_image",
1,
is_video=False,
output_format="png",
) == [
"5090/unit_image_1gpu.png",
"5090/unit_image_1gpu.jpg",
"5090/unit_image_1gpu.webp",
"unit_image_1gpu.png",
"unit_image_1gpu.jpg",
"unit_image_1gpu.webp",
]
def test_threshold_metadata_merges_platform_override():
metadata = test_utils._merge_threshold_metadata(
{
"cases": {
"case_a": {
"clip_threshold": 0.9,
"ssim_threshold": 0.9,
"psnr_threshold": 20.0,
"mean_abs_diff_threshold": 10.0,
}
},
"default_clip_threshold_image": 0.92,
},
{
"cases": {
"case_a": {
"clip_threshold": 0.8,
"ssim_threshold": 0.7,
"psnr_threshold": 12.0,
"mean_abs_diff_threshold": 20.0,
}
}
},
)
assert metadata["default_clip_threshold_image"] == 0.92
assert metadata["cases"]["case_a"]["psnr_threshold"] == 12.0
def test_pixel_metrics_identical_image():
image = _solid_image(128)
python/sglang/multimodal_gen/test/unit/test_turbo_wan_backend.py
+61
View File
@@ -0,0 +1,61 @@
import unittest
from sglang.multimodal_gen.runtime.layers.attention.turbo_layer import (
_resolve_turbo_wan_sparse_backend,
)
from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum
class TestTurboWanBackendSelection(unittest.TestCase):
def test_non_sparse_requested_backend_falls_back_to_attention_type(self):
selected, warning = _resolve_turbo_wan_sparse_backend(
attention_type="sla",
requested_attention_backend="fa",
)
self.assertEqual(selected, AttentionBackendEnum.SLA_ATTN)
self.assertIsNotNone(warning)
self.assertIn("TurboWan only supports", warning)
self.assertIn("attention_backend='fa'", warning)
def test_sagesla_attention_type_prefers_sage_sparse_backend(self):
selected, warning = _resolve_turbo_wan_sparse_backend(
attention_type="sagesla",
requested_attention_backend="torch_sdpa",
)
self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN)
self.assertIsNotNone(warning)
def test_requested_sparse_backend_is_honored(self):
selected, warning = _resolve_turbo_wan_sparse_backend(
attention_type="sla",
requested_attention_backend="sage_sla_attn",
)
self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN)
self.assertIsNone(warning)
def test_supported_backend_filter_is_respected(self):
selected, warning = _resolve_turbo_wan_sparse_backend(
attention_type="sla",
requested_attention_backend=None,
supported_attention_backends={AttentionBackendEnum.SAGE_SLA_ATTN},
)
self.assertEqual(selected, AttentionBackendEnum.SAGE_SLA_ATTN)
self.assertIsNone(warning)
def test_empty_supported_backend_intersection_keeps_turbowan_choices(self):
selected, warning = _resolve_turbo_wan_sparse_backend(
attention_type="sla",
requested_attention_backend=None,
supported_attention_backends={AttentionBackendEnum.FA},
)
self.assertEqual(selected, AttentionBackendEnum.SLA_ATTN)
self.assertIsNone(warning)
if __name__ == "__main__":
unittest.main()