[diffusion] CI: prefer official diffusion consistency GT (#29831)
This commit is contained in:
@@ -154,7 +154,9 @@ async def run_async_client_warmup(
|
|||||||
raise RuntimeError(response.error)
|
raise RuntimeError(response.error)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
if fail_open:
|
if fail_open:
|
||||||
logger.warning("Synthetic server warmup failed; continuing startup: %s", e)
|
logger.warning(
|
||||||
|
"Synthetic server warmup failed; continuing startup", exc_info=True
|
||||||
|
)
|
||||||
return
|
return
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -278,7 +280,7 @@ class SchedulerWarmupMixin:
|
|||||||
self._logged_server_ready_after_warmup = True
|
self._logged_server_ready_after_warmup = True
|
||||||
else:
|
else:
|
||||||
warmup_desc = self._format_warmup_req(req_or_group)
|
warmup_desc = self._format_warmup_req(req_or_group)
|
||||||
logger.info(f"{warmup_desc} processing failed")
|
logger.warning("%s processing failed: %s", warmup_desc, output_batch.error)
|
||||||
|
|
||||||
def process_received_reqs_with_req_based_warmup(
|
def process_received_reqs_with_req_based_warmup(
|
||||||
self, recv_reqs: list[tuple[bytes, Any]]
|
self, recv_reqs: list[tuple[bytes, Any]]
|
||||||
|
|||||||
@@ -21,13 +21,13 @@ from sglang.multimodal_gen.test.run_suite import (
|
|||||||
SUITES,
|
SUITES,
|
||||||
PartitionItem,
|
PartitionItem,
|
||||||
_maybe_pin_update_weights_model_pair,
|
_maybe_pin_update_weights_model_pair,
|
||||||
collect_test_items,
|
|
||||||
get_case_est_time,
|
get_case_est_time,
|
||||||
get_suite_files_rel,
|
get_suite_files_rel,
|
||||||
parse_partition_plan,
|
parse_partition_plan,
|
||||||
partition_items_by_lpt,
|
partition_items_by_lpt,
|
||||||
run_pytest,
|
run_pytest,
|
||||||
)
|
)
|
||||||
|
from sglang.multimodal_gen.test.runner.pytest_runner import collect_test_items
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|||||||
@@ -10,8 +10,8 @@
|
|||||||
"flux_image_t2i": {
|
"flux_image_t2i": {
|
||||||
"clip_threshold": 0.97,
|
"clip_threshold": 0.97,
|
||||||
"ssim_threshold": 0.95,
|
"ssim_threshold": 0.95,
|
||||||
"psnr_threshold": 30.0,
|
"psnr_threshold": 24.0,
|
||||||
"mean_abs_diff_threshold": 4.0
|
"mean_abs_diff_threshold": 5.5
|
||||||
},
|
},
|
||||||
"flux1_modelopt_fp8_t2i": {
|
"flux1_modelopt_fp8_t2i": {
|
||||||
"clip_threshold": 0.96,
|
"clip_threshold": 0.96,
|
||||||
@@ -38,16 +38,16 @@
|
|||||||
"mean_abs_diff_threshold": 9.0
|
"mean_abs_diff_threshold": 9.0
|
||||||
},
|
},
|
||||||
"flux_2_klein_image_t2i": {
|
"flux_2_klein_image_t2i": {
|
||||||
"clip_threshold": 0.98,
|
"clip_threshold": 0.97,
|
||||||
"ssim_threshold": 0.95,
|
"ssim_threshold": 0.83,
|
||||||
"psnr_threshold": 17.0,
|
"psnr_threshold": 17.0,
|
||||||
"mean_abs_diff_threshold": 4.0
|
"mean_abs_diff_threshold": 12.5
|
||||||
},
|
},
|
||||||
"zimage_image_t2i": {
|
"zimage_image_t2i": {
|
||||||
"clip_threshold": 0.97,
|
"clip_threshold": 0.97,
|
||||||
"ssim_threshold": 0.95,
|
"ssim_threshold": 0.86,
|
||||||
"psnr_threshold": 30.0,
|
"psnr_threshold": 19.9,
|
||||||
"mean_abs_diff_threshold": 4.0
|
"mean_abs_diff_threshold": 8.5
|
||||||
},
|
},
|
||||||
"zimage_image_t2i_multi_lora": {
|
"zimage_image_t2i_multi_lora": {
|
||||||
"clip_threshold": 0.98,
|
"clip_threshold": 0.98,
|
||||||
@@ -74,10 +74,10 @@
|
|||||||
"mean_abs_diff_threshold": 4.0
|
"mean_abs_diff_threshold": 4.0
|
||||||
},
|
},
|
||||||
"flux_2_ti2i": {
|
"flux_2_ti2i": {
|
||||||
"clip_threshold": 0.98,
|
"clip_threshold": 0.97,
|
||||||
"ssim_threshold": 0.95,
|
"ssim_threshold": 0.88,
|
||||||
"psnr_threshold": 19.5,
|
"psnr_threshold": 19.5,
|
||||||
"mean_abs_diff_threshold": 4.0
|
"mean_abs_diff_threshold": 13.5
|
||||||
},
|
},
|
||||||
"zimage_image_t2i_fp8": {
|
"zimage_image_t2i_fp8": {
|
||||||
"clip_threshold": 0.97,
|
"clip_threshold": 0.97,
|
||||||
@@ -92,16 +92,16 @@
|
|||||||
"mean_abs_diff_threshold": 26.4
|
"mean_abs_diff_threshold": 26.4
|
||||||
},
|
},
|
||||||
"qwen_image_edit_ti2i": {
|
"qwen_image_edit_ti2i": {
|
||||||
"clip_threshold": 0.97,
|
"clip_threshold": 0.96,
|
||||||
"ssim_threshold": 0.95,
|
"ssim_threshold": 0.94,
|
||||||
"psnr_threshold": 30.0,
|
"psnr_threshold": 25.4,
|
||||||
"mean_abs_diff_threshold": 4.0
|
"mean_abs_diff_threshold": 10.0
|
||||||
},
|
},
|
||||||
"qwen_image_edit_2511_ti2i": {
|
"qwen_image_edit_2511_ti2i": {
|
||||||
"clip_threshold": 0.97,
|
"clip_threshold": 0.96,
|
||||||
"ssim_threshold": 0.95,
|
"ssim_threshold": 0.83,
|
||||||
"psnr_threshold": 30.0,
|
"psnr_threshold": 20.5,
|
||||||
"mean_abs_diff_threshold": 4.1
|
"mean_abs_diff_threshold": 17.0
|
||||||
},
|
},
|
||||||
"qwen_image_layered_i2i": {
|
"qwen_image_layered_i2i": {
|
||||||
"clip_threshold": 0.97,
|
"clip_threshold": 0.97,
|
||||||
@@ -117,9 +117,9 @@
|
|||||||
},
|
},
|
||||||
"wan2_1_t2v_1.3b": {
|
"wan2_1_t2v_1.3b": {
|
||||||
"clip_threshold": 0.97,
|
"clip_threshold": 0.97,
|
||||||
"ssim_threshold": 0.90,
|
"ssim_threshold": 0.85,
|
||||||
"psnr_threshold": 30.0,
|
"psnr_threshold": 25.0,
|
||||||
"mean_abs_diff_threshold": 5.6
|
"mean_abs_diff_threshold": 8.0
|
||||||
},
|
},
|
||||||
"cosmos3_nano_t2v": {
|
"cosmos3_nano_t2v": {
|
||||||
"clip_threshold": 0.91,
|
"clip_threshold": 0.91,
|
||||||
@@ -278,10 +278,10 @@
|
|||||||
"mean_abs_diff_threshold": 5.5
|
"mean_abs_diff_threshold": 5.5
|
||||||
},
|
},
|
||||||
"ltx_2_3_hq_pipeline": {
|
"ltx_2_3_hq_pipeline": {
|
||||||
"clip_threshold": 0.97,
|
"clip_threshold": 0.995,
|
||||||
"ssim_threshold": 0.88,
|
"ssim_threshold": 0.995,
|
||||||
"psnr_threshold": 26.3,
|
"psnr_threshold": 50.0,
|
||||||
"mean_abs_diff_threshold": 5.8
|
"mean_abs_diff_threshold": 0.5
|
||||||
},
|
},
|
||||||
"ltx_2_3_two_stage_ti2v_2gpus": {
|
"ltx_2_3_two_stage_ti2v_2gpus": {
|
||||||
"clip_threshold": 0.55,
|
"clip_threshold": 0.55,
|
||||||
|
|||||||
@@ -746,10 +746,10 @@
|
|||||||
"7": 64.42,
|
"7": 64.42,
|
||||||
"8": 64.35
|
"8": 64.35
|
||||||
},
|
},
|
||||||
"expected_e2e_ms": 961.86,
|
"expected_e2e_ms": 1425.0,
|
||||||
"expected_avg_denoise_ms": 57.95,
|
"expected_avg_denoise_ms": 57.95,
|
||||||
"expected_median_denoise_ms": 64.35,
|
"expected_median_denoise_ms": 64.35,
|
||||||
"estimated_full_test_time_s": 59.9
|
"estimated_full_test_time_s": 75.0
|
||||||
},
|
},
|
||||||
"qwen_image_edit_ti2i": {
|
"qwen_image_edit_ti2i": {
|
||||||
"stages_ms": {
|
"stages_ms": {
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
SGL_TEST_FILES_CI_DATA_REVISION = "702d939e23f17b42183329dace60f221d2587056"
|
SGL_TEST_FILES_CI_DATA_REVISION = "c28c11c24125b009f9e602fd5ae0d0ddcdc11d36"
|
||||||
|
|
||||||
if current_platform.is_npu():
|
if current_platform.is_npu():
|
||||||
SGL_TEST_FILES_CI_DATA_REVISION = "670d66a8a290b62c0c3c077b3e9b0f4a4d9a44e7"
|
SGL_TEST_FILES_CI_DATA_REVISION = "670d66a8a290b62c0c3c077b3e9b0f4a4d9a44e7"
|
||||||
@@ -64,21 +64,6 @@ if current_platform.is_npu():
|
|||||||
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE_ASCEND
|
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE_ASCEND
|
||||||
)
|
)
|
||||||
|
|
||||||
SGL_TEST_FILES_CONSISTENCY_GT_BASES = (
|
|
||||||
SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE,
|
|
||||||
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE,
|
|
||||||
SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE_ASCEND,
|
|
||||||
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE_ASCEND,
|
|
||||||
)
|
|
||||||
# LTX cases listed here compare against official-generated GT.
|
|
||||||
SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_CASES = frozenset(
|
|
||||||
{
|
|
||||||
"ltx_2.3_one_stage_ti2v",
|
|
||||||
"ltx_2.3_two_stage_t2v_2gpus",
|
|
||||||
"ltx_2_3_two_stage_ti2v_2gpus",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
CONSISTENCY_PLATFORM_ENV = "SGLANG_DIFFUSION_CONSISTENCY_PLATFORM"
|
CONSISTENCY_PLATFORM_ENV = "SGLANG_DIFFUSION_CONSISTENCY_PLATFORM"
|
||||||
CONSISTENCY_THRESHOLD_DIR = (
|
CONSISTENCY_THRESHOLD_DIR = (
|
||||||
Path(__file__).resolve().parent / "server" / "consistency_thresholds"
|
Path(__file__).resolve().parent / "server" / "consistency_thresholds"
|
||||||
@@ -110,6 +95,17 @@ DEFAULT_PSNR_THRESHOLD_VIDEO = 24.0
|
|||||||
DEFAULT_MEAN_ABS_DIFF_THRESHOLD_VIDEO = 10.0
|
DEFAULT_MEAN_ABS_DIFF_THRESHOLD_VIDEO = 10.0
|
||||||
_clip_model_cache: dict[str, Any] = {}
|
_clip_model_cache: dict[str, Any] = {}
|
||||||
_consistency_gt_cache: dict[str, Any] = {}
|
_consistency_gt_cache: dict[str, Any] = {}
|
||||||
|
_official_consistency_gt_outputs_cache: dict[str, frozenset[str]] | None = None
|
||||||
|
OFFICIAL_CONSISTENCY_GT_SKIP_CASES = frozenset(
|
||||||
|
{
|
||||||
|
# Official references for these cases need regeneration or parity triage.
|
||||||
|
# Prefer existing sglang-generated GT instead of relaxing thresholds over
|
||||||
|
# large semantic/content mismatches.
|
||||||
|
"ltx_2_3_hq_pipeline",
|
||||||
|
"ltx_2_two_stage_t2v",
|
||||||
|
"qwen_image_edit_2509_ti2i",
|
||||||
|
}
|
||||||
|
)
|
||||||
# Case keys whose remote GT has been positively confirmed present. Cached so a
|
# Case keys whose remote GT has been positively confirmed present. Cached so a
|
||||||
# case that probes GT existence more than once in a single run — e.g. a
|
# case that probes GT existence more than once in a single run — e.g. a
|
||||||
# consistency check followed by the LoRA basic-API check, which re-validates
|
# consistency check followed by the LoRA basic-API check, which re-validates
|
||||||
@@ -1104,25 +1100,95 @@ def _remote_consistency_gt_candidates(
|
|||||||
return [(filename, f"{base_url}/{filename}") for filename in filenames]
|
return [(filename, f"{base_url}/{filename}") for filename in filenames]
|
||||||
|
|
||||||
|
|
||||||
def _remote_consistency_gt_candidate_sets(
|
|
||||||
base_url: str,
|
|
||||||
case_id: str,
|
|
||||||
num_gpus: int,
|
|
||||||
is_video: bool,
|
|
||||||
output_format: str | None = None,
|
|
||||||
) -> list[list[tuple[str, str]]]:
|
|
||||||
return [
|
|
||||||
[(filename, f"{base_url}/{filename}") for filename in filenames]
|
|
||||||
for filenames in get_consistency_gt_candidate_sets(
|
|
||||||
case_id, num_gpus, is_video, output_format
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _is_ascend_consistency_case(case_id: str) -> bool:
|
def _is_ascend_consistency_case(case_id: str) -> bool:
|
||||||
return "npu" in case_id
|
return "npu" in case_id
|
||||||
|
|
||||||
|
|
||||||
|
def _load_official_consistency_gt_outputs() -> dict[str, frozenset[str]]:
|
||||||
|
"""Return case_id -> declared official GT outputs from the pinned ci-data map."""
|
||||||
|
global _official_consistency_gt_outputs_cache
|
||||||
|
if _official_consistency_gt_outputs_cache is not None:
|
||||||
|
return _official_consistency_gt_outputs_cache
|
||||||
|
|
||||||
|
url = f"{SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE}/case_map.json"
|
||||||
|
outputs_by_case: dict[str, frozenset[str]] = {}
|
||||||
|
try:
|
||||||
|
resp = requests.get(url, timeout=30)
|
||||||
|
try:
|
||||||
|
if resp.status_code == 200:
|
||||||
|
data = resp.json()
|
||||||
|
else:
|
||||||
|
data = {}
|
||||||
|
logger.warning(
|
||||||
|
"Failed to load official consistency GT case map from %s: HTTP %s",
|
||||||
|
url,
|
||||||
|
resp.status_code,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
resp.close()
|
||||||
|
except (ValueError, requests.RequestException) as exc:
|
||||||
|
data = {}
|
||||||
|
logger.warning(
|
||||||
|
"Failed to load official consistency GT case map from %s: %s",
|
||||||
|
url,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
|
||||||
|
cases = data.get("cases", {}) if isinstance(data, dict) else {}
|
||||||
|
if isinstance(cases, dict):
|
||||||
|
for case_id, metadata in cases.items():
|
||||||
|
outputs = metadata.get("outputs", []) if isinstance(metadata, dict) else []
|
||||||
|
if isinstance(outputs, list):
|
||||||
|
outputs_by_case[str(case_id)] = frozenset(str(item) for item in outputs)
|
||||||
|
|
||||||
|
_official_consistency_gt_outputs_cache = outputs_by_case
|
||||||
|
return outputs_by_case
|
||||||
|
|
||||||
|
|
||||||
|
def _official_consistency_gt_outputs_for_case(case_id: str) -> frozenset[str]:
|
||||||
|
return _load_official_consistency_gt_outputs().get(case_id, frozenset())
|
||||||
|
|
||||||
|
|
||||||
|
def _is_official_consistency_gt_base_url(base_url: str) -> bool:
|
||||||
|
return base_url in (
|
||||||
|
SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE,
|
||||||
|
SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE_ASCEND,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _official_consistency_gt_candidate_is_declared(case_id: str, filename: str) -> bool:
|
||||||
|
return filename in _official_consistency_gt_outputs_for_case(case_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _remote_consistency_gt_base_urls(case_id: str) -> tuple[str, ...]:
|
||||||
|
if case_id in OFFICIAL_CONSISTENCY_GT_SKIP_CASES:
|
||||||
|
if _is_ascend_consistency_case(case_id) or current_platform.is_npu():
|
||||||
|
return (
|
||||||
|
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE_ASCEND,
|
||||||
|
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE,
|
||||||
|
)
|
||||||
|
return (SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE,)
|
||||||
|
has_declared_official_gt = bool(_official_consistency_gt_outputs_for_case(case_id))
|
||||||
|
if _is_ascend_consistency_case(case_id) or current_platform.is_npu():
|
||||||
|
if has_declared_official_gt:
|
||||||
|
return (
|
||||||
|
SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE_ASCEND,
|
||||||
|
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE_ASCEND,
|
||||||
|
SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE,
|
||||||
|
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE,
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE_ASCEND,
|
||||||
|
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE,
|
||||||
|
)
|
||||||
|
if has_declared_official_gt:
|
||||||
|
return (
|
||||||
|
SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE,
|
||||||
|
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE,
|
||||||
|
)
|
||||||
|
return (SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE,)
|
||||||
|
|
||||||
|
|
||||||
def _remote_file_exists(url: str) -> bool | None:
|
def _remote_file_exists(url: str) -> bool | None:
|
||||||
"""Probe whether a remote GT file exists, robust to transient failures."""
|
"""Probe whether a remote GT file exists, robust to transient failures."""
|
||||||
attempts = 5
|
attempts = 5
|
||||||
@@ -1191,21 +1257,21 @@ def _find_remote_consistency_gt_files(
|
|||||||
is_video: bool,
|
is_video: bool,
|
||||||
output_format: str | None = None,
|
output_format: str | None = None,
|
||||||
) -> list[tuple[str, str]]:
|
) -> list[tuple[str, str]]:
|
||||||
if _is_ascend_consistency_case(case_id):
|
for filenames in get_consistency_gt_candidate_sets(
|
||||||
bases = (
|
case_id, num_gpus, is_video, output_format
|
||||||
SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE_ASCEND,
|
):
|
||||||
SGL_TEST_FILES_CONSISTENCY_GT_BASE,
|
for base_url in _remote_consistency_gt_base_urls(case_id):
|
||||||
)
|
candidates = [
|
||||||
elif case_id in SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_CASES:
|
(filename, f"{base_url}/{filename}") for filename in filenames
|
||||||
bases = SGL_TEST_FILES_CONSISTENCY_GT_BASES
|
]
|
||||||
else:
|
if _is_official_consistency_gt_base_url(base_url):
|
||||||
# Avoid accidentally comparing non-comparable CI cases against official GT.
|
candidates = [
|
||||||
bases = (SGL_TEST_FILES_CONSISTENCY_GT_BASE,)
|
(filename, url)
|
||||||
for base_url in bases:
|
for filename, url in candidates
|
||||||
candidate_sets = _remote_consistency_gt_candidate_sets(
|
if _official_consistency_gt_candidate_is_declared(case_id, filename)
|
||||||
base_url, case_id, num_gpus, is_video, output_format
|
]
|
||||||
)
|
if not candidates or (is_video and len(candidates) != len(filenames)):
|
||||||
for candidates in candidate_sets:
|
continue
|
||||||
if is_video:
|
if is_video:
|
||||||
exists = [_remote_file_exists(url) for _, url in candidates]
|
exists = [_remote_file_exists(url) for _, url in candidates]
|
||||||
if all(status is not False for status in exists):
|
if all(status is not False for status in exists):
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import math
|
import math
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
from sglang.multimodal_gen.test import test_utils
|
from sglang.multimodal_gen.test import test_utils
|
||||||
from sglang.multimodal_gen.test.test_utils import (
|
from sglang.multimodal_gen.test.test_utils import (
|
||||||
@@ -18,6 +19,19 @@ def _solid_image(value: int, size: int = 32) -> np.ndarray:
|
|||||||
return np.full((size, size, 3), value, dtype=np.uint8)
|
return np.full((size, size, 3), value, dtype=np.uint8)
|
||||||
|
|
||||||
|
|
||||||
|
def _set_official_gt_outputs(monkeypatch, outputs_by_case):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
test_utils,
|
||||||
|
"_official_consistency_gt_outputs_for_case",
|
||||||
|
lambda case_id: frozenset(outputs_by_case.get(case_id, ())),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _disable_remote_official_gt_case_map(monkeypatch):
|
||||||
|
_set_official_gt_outputs(monkeypatch, {})
|
||||||
|
|
||||||
|
|
||||||
def test_consistency_gt_urls_are_pinned_to_ci_data_revision():
|
def test_consistency_gt_urls_are_pinned_to_ci_data_revision():
|
||||||
revision_path = f"/ci-data/{test_utils.SGL_TEST_FILES_CI_DATA_REVISION}/"
|
revision_path = f"/ci-data/{test_utils.SGL_TEST_FILES_CI_DATA_REVISION}/"
|
||||||
|
|
||||||
@@ -54,6 +68,297 @@ def test_remote_video_gt_candidates_survive_inconclusive_probe(monkeypatch):
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_remote_image_gt_prefers_official_when_present(monkeypatch):
|
||||||
|
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "h100")
|
||||||
|
official_prefix = test_utils.SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE + "/"
|
||||||
|
expected_filename = f"unit_image_1gpu.{test_utils.output_format_to_ext(None)}"
|
||||||
|
_set_official_gt_outputs(monkeypatch, {"unit_image": [expected_filename]})
|
||||||
|
monkeypatch.setattr(
|
||||||
|
test_utils,
|
||||||
|
"_remote_file_exists",
|
||||||
|
lambda url: url.startswith(official_prefix),
|
||||||
|
)
|
||||||
|
|
||||||
|
files = test_utils._find_remote_consistency_gt_files(
|
||||||
|
"unit_image",
|
||||||
|
1,
|
||||||
|
is_video=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert files == [
|
||||||
|
(
|
||||||
|
expected_filename,
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE}"
|
||||||
|
f"/{expected_filename}"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_remote_image_gt_ignores_unmapped_official_file(monkeypatch):
|
||||||
|
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "h100")
|
||||||
|
official_prefix = test_utils.SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE + "/"
|
||||||
|
sglang_prefix = test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE + "/"
|
||||||
|
expected_filename = f"unit_image_1gpu.{test_utils.output_format_to_ext(None)}"
|
||||||
|
_set_official_gt_outputs(monkeypatch, {})
|
||||||
|
monkeypatch.setattr(
|
||||||
|
test_utils,
|
||||||
|
"_remote_file_exists",
|
||||||
|
lambda url: url.startswith(official_prefix) or url.startswith(sglang_prefix),
|
||||||
|
)
|
||||||
|
|
||||||
|
files = test_utils._find_remote_consistency_gt_files(
|
||||||
|
"unit_image",
|
||||||
|
1,
|
||||||
|
is_video=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert files == [
|
||||||
|
(
|
||||||
|
expected_filename,
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
f"/{expected_filename}"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_remote_video_gt_ignores_unmapped_official_files(monkeypatch):
|
||||||
|
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "h100")
|
||||||
|
official_prefix = test_utils.SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE + "/"
|
||||||
|
sglang_prefix = test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE + "/"
|
||||||
|
case_id = "unit_video"
|
||||||
|
_set_official_gt_outputs(monkeypatch, {})
|
||||||
|
monkeypatch.setattr(
|
||||||
|
test_utils,
|
||||||
|
"_remote_file_exists",
|
||||||
|
lambda url: url.startswith(official_prefix) or url.startswith(sglang_prefix),
|
||||||
|
)
|
||||||
|
|
||||||
|
files = test_utils._find_remote_consistency_gt_files(case_id, 2, is_video=True)
|
||||||
|
|
||||||
|
assert files == [
|
||||||
|
(
|
||||||
|
f"{case_id}_2gpu_frame_0.png",
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
f"/{case_id}_2gpu_frame_0.png"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f"{case_id}_2gpu_frame_mid.png",
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
f"/{case_id}_2gpu_frame_mid.png"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f"{case_id}_2gpu_frame_last.png",
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
f"/{case_id}_2gpu_frame_last.png"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_ltx_hq_remote_gt_uses_sglang_generated_when_official_declared(monkeypatch):
|
||||||
|
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "h100")
|
||||||
|
case_id = "ltx_2_3_hq_pipeline"
|
||||||
|
filenames = [
|
||||||
|
f"{case_id}_1gpu_frame_0.png",
|
||||||
|
f"{case_id}_1gpu_frame_mid.png",
|
||||||
|
f"{case_id}_1gpu_frame_last.png",
|
||||||
|
]
|
||||||
|
_set_official_gt_outputs(monkeypatch, {case_id: filenames})
|
||||||
|
monkeypatch.setattr(test_utils, "_remote_file_exists", lambda url: True)
|
||||||
|
|
||||||
|
files = test_utils._find_remote_consistency_gt_files(case_id, 1, is_video=True)
|
||||||
|
|
||||||
|
assert files == [
|
||||||
|
(filename, f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}/{filename}")
|
||||||
|
for filename in filenames
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_remote_image_gt_falls_back_to_sglang_when_official_missing(monkeypatch):
|
||||||
|
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "h100")
|
||||||
|
sglang_prefix = test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE + "/"
|
||||||
|
expected_filename = f"unit_image_1gpu.{test_utils.output_format_to_ext(None)}"
|
||||||
|
monkeypatch.setattr(
|
||||||
|
test_utils,
|
||||||
|
"_remote_file_exists",
|
||||||
|
lambda url: url.startswith(sglang_prefix),
|
||||||
|
)
|
||||||
|
|
||||||
|
files = test_utils._find_remote_consistency_gt_files(
|
||||||
|
"unit_image",
|
||||||
|
1,
|
||||||
|
is_video=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert files == [
|
||||||
|
(
|
||||||
|
expected_filename,
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
f"/{expected_filename}"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_remote_image_gt_skips_official_for_quarantined_case(monkeypatch):
|
||||||
|
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "h100")
|
||||||
|
official_prefix = test_utils.SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE + "/"
|
||||||
|
sglang_prefix = test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE + "/"
|
||||||
|
case_id = "qwen_image_edit_2509_ti2i"
|
||||||
|
expected_filename = f"{case_id}_1gpu.{test_utils.output_format_to_ext(None)}"
|
||||||
|
monkeypatch.setattr(
|
||||||
|
test_utils,
|
||||||
|
"_remote_file_exists",
|
||||||
|
lambda url: url.startswith(official_prefix) or url.startswith(sglang_prefix),
|
||||||
|
)
|
||||||
|
|
||||||
|
files = test_utils._find_remote_consistency_gt_files(
|
||||||
|
case_id,
|
||||||
|
1,
|
||||||
|
is_video=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert files == [
|
||||||
|
(
|
||||||
|
expected_filename,
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
f"/{expected_filename}"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_remote_platform_video_gt_prefers_platform_sglang_before_default_official(
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "5090")
|
||||||
|
sglang_platform_prefix = (
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}/5090/"
|
||||||
|
)
|
||||||
|
official_default_prefix = (
|
||||||
|
f"{test_utils.SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE}/unit_video_1gpu_"
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
test_utils,
|
||||||
|
"_remote_file_exists",
|
||||||
|
lambda url: url.startswith(sglang_platform_prefix)
|
||||||
|
or url.startswith(official_default_prefix),
|
||||||
|
)
|
||||||
|
|
||||||
|
files = test_utils._find_remote_consistency_gt_files(
|
||||||
|
"unit_video",
|
||||||
|
1,
|
||||||
|
is_video=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert files == [
|
||||||
|
(
|
||||||
|
"5090/unit_video_1gpu_frame_0.png",
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
"/5090/unit_video_1gpu_frame_0.png"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"5090/unit_video_1gpu_frame_mid.png",
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
"/5090/unit_video_1gpu_frame_mid.png"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"5090/unit_video_1gpu_frame_last.png",
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
"/5090/unit_video_1gpu_frame_last.png"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_remote_video_gt_skips_official_for_quarantined_case(monkeypatch):
|
||||||
|
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "h100")
|
||||||
|
official_prefix = test_utils.SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE + "/"
|
||||||
|
sglang_prefix = test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE + "/"
|
||||||
|
case_id = "ltx_2_two_stage_t2v"
|
||||||
|
monkeypatch.setattr(
|
||||||
|
test_utils,
|
||||||
|
"_remote_file_exists",
|
||||||
|
lambda url: url.startswith(official_prefix) or url.startswith(sglang_prefix),
|
||||||
|
)
|
||||||
|
|
||||||
|
files = test_utils._find_remote_consistency_gt_files(
|
||||||
|
case_id,
|
||||||
|
2,
|
||||||
|
is_video=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert files == [
|
||||||
|
(
|
||||||
|
f"{case_id}_2gpu_frame_0.png",
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
f"/{case_id}_2gpu_frame_0.png"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f"{case_id}_2gpu_frame_mid.png",
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
f"/{case_id}_2gpu_frame_mid.png"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f"{case_id}_2gpu_frame_last.png",
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE}"
|
||||||
|
f"/{case_id}_2gpu_frame_last.png"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_remote_npu_image_gt_prefers_official_ascend_when_present(monkeypatch):
|
||||||
|
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "h100")
|
||||||
|
official_ascend_prefix = (
|
||||||
|
test_utils.SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE_ASCEND + "/"
|
||||||
|
)
|
||||||
|
expected_filename = f"unit_npu_image_1gpu.{test_utils.output_format_to_ext(None)}"
|
||||||
|
_set_official_gt_outputs(monkeypatch, {"unit_npu_image": [expected_filename]})
|
||||||
|
monkeypatch.setattr(
|
||||||
|
test_utils,
|
||||||
|
"_remote_file_exists",
|
||||||
|
lambda url: url.startswith(official_ascend_prefix),
|
||||||
|
)
|
||||||
|
|
||||||
|
files = test_utils._find_remote_consistency_gt_files(
|
||||||
|
"unit_npu_image",
|
||||||
|
1,
|
||||||
|
is_video=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert files == [
|
||||||
|
(
|
||||||
|
expected_filename,
|
||||||
|
(
|
||||||
|
f"{test_utils.SGL_TEST_FILES_OFFICIAL_CONSISTENCY_GT_BASE_ASCEND}"
|
||||||
|
f"/{expected_filename}"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_platform_gt_candidates_prefer_platform_then_default(monkeypatch):
|
def test_platform_gt_candidates_prefer_platform_then_default(monkeypatch):
|
||||||
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "5090")
|
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "5090")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user