[diffusion] CI: batch performance failure-policy runner scenarios (#44508)
This commit is contained in:
@@ -197,9 +197,9 @@ def test_retry_estimate_excludes_successful_cases(tmp_path):
|
||||
assert _estimate_failed_test_time(None, 130) == 130
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"problem",
|
||||
[
|
||||
def test_performance_failure_survives_real_pytest_runner(tmp_path):
|
||||
# share imports across failure scenarios while exercising all seven real attempts
|
||||
problems = [
|
||||
"regression",
|
||||
"missing_baseline",
|
||||
"missing_record",
|
||||
@@ -209,10 +209,7 @@ def test_retry_estimate_excludes_successful_cases(tmp_path):
|
||||
"e2e_only_missing_baseline",
|
||||
"e2e_only_zero_baseline",
|
||||
"e2e_only_nan_baseline",
|
||||
],
|
||||
)
|
||||
def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
|
||||
# exercise the validator, request loop, pytest output and retry classifier together
|
||||
]
|
||||
test_file = tmp_path / "test_guard.py"
|
||||
test_file.write_text(
|
||||
textwrap.dedent(
|
||||
@@ -226,7 +223,7 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
|
||||
DiffusionSamplingParams, DiffusionServerArgs, DiffusionTestCase, ScenarioConfig,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("case_id", ["threshold_guard"])
|
||||
@pytest.mark.parametrize("case_id", PROBLEMS)
|
||||
def test_guard(case_id, monkeypatch, tmp_path):
|
||||
server = common.DiffusionServerBase()
|
||||
server._perf_results = []
|
||||
@@ -236,14 +233,14 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
|
||||
DiffusionSamplingParams(prompt="test"),
|
||||
run_lora_basic_api_check=True, perf_repeat_requests=2,
|
||||
run_consistency_check=False, run_models_api_check=False,
|
||||
run_perf_check=not PROBLEM.startswith("e2e_only_") and PROBLEM not in ("missing_record", "missing_e2e", "missing_log"),
|
||||
run_perf_check=not case_id.startswith("e2e_only_") and case_id not in ("missing_record", "missing_e2e", "missing_log"),
|
||||
)
|
||||
scenario = ScenarioConfig({}, {}, 1000, 100, 100, expected_load_ms=100)
|
||||
if PROBLEM == "e2e_only_zero_baseline":
|
||||
if case_id == "e2e_only_zero_baseline":
|
||||
scenario.expected_e2e_ms = 0
|
||||
if PROBLEM == "e2e_only_nan_baseline":
|
||||
if case_id == "e2e_only_nan_baseline":
|
||||
scenario.expected_e2e_ms = float("nan")
|
||||
if PROBLEM in ("missing_baseline", "e2e_only_missing_baseline"):
|
||||
if case_id in ("missing_baseline", "e2e_only_missing_baseline"):
|
||||
monkeypatch.delitem(common.BASELINE_CONFIG.scenarios, case_id, raising=False)
|
||||
else:
|
||||
monkeypatch.setitem(common.BASELINE_CONFIG.scenarios, case_id, scenario)
|
||||
@@ -254,15 +251,15 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
|
||||
lora_checks = []
|
||||
def collect(*args, **kwargs):
|
||||
requests.append(1)
|
||||
if PROBLEM == "missing_record":
|
||||
if case_id == "missing_record":
|
||||
return None, b""
|
||||
return RequestPerfRecord(
|
||||
request_id="guard", commit_hash="test", tag="guard",
|
||||
stages=[], steps=[100],
|
||||
total_duration_ms=None if PROBLEM == "missing_e2e" else 2000,
|
||||
total_duration_ms=None if case_id == "missing_e2e" else 2000,
|
||||
), b""
|
||||
context = SimpleNamespace(load_time_ms=100)
|
||||
if PROBLEM == "missing_log":
|
||||
if case_id == "missing_log":
|
||||
log_path = tmp_path / "empty-perf.jsonl"
|
||||
log_path.write_text("")
|
||||
context = SimpleNamespace(perf_log_path=log_path, load_time_ms=100)
|
||||
@@ -284,13 +281,15 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
|
||||
try:
|
||||
server._test_diffusion_generation_impl(case, context)
|
||||
finally:
|
||||
print(f"GUARD_REQUESTS={len(requests)} LORA_CHECKS={len(lora_checks)}")
|
||||
print(f"RETAINED_METRICS={len(server._perf_results)}")
|
||||
print(
|
||||
f"GUARD_CASE={case_id} GUARD_REQUESTS={len(requests)} "
|
||||
f"LORA_CHECKS={len(lora_checks)} RETAINED_METRICS={len(server._perf_results)}"
|
||||
)
|
||||
|
||||
def test_unrelated_timeout():
|
||||
raise TimeoutError("independent infrastructure failure")
|
||||
"""
|
||||
).replace("PROBLEM", repr(problem))
|
||||
).replace("PROBLEMS", repr(problems))
|
||||
)
|
||||
report = tmp_path / "junit.xml"
|
||||
env = os.environ.copy()
|
||||
@@ -315,9 +314,15 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
|
||||
output = result.stdout + result.stderr
|
||||
assert result.returncode == 1, output
|
||||
assert "[performance]" in output, output
|
||||
assert "GUARD_REQUESTS=1 LORA_CHECKS=0" in output, output
|
||||
retained = 0 if problem in {"missing_record", "missing_e2e", "missing_log"} else 1
|
||||
assert f"RETAINED_METRICS={retained}" in output, output
|
||||
assert output.count("Starting pytest attempt") == 7, output
|
||||
assert "Max retry exceeded (6)" in output, output
|
||||
assert "'threshold_guard': 'fail'" in output, output
|
||||
for problem in problems:
|
||||
retained = (
|
||||
0 if problem in {"missing_record", "missing_e2e", "missing_log"} else 1
|
||||
)
|
||||
marker = (
|
||||
f"GUARD_CASE={problem} GUARD_REQUESTS=1 "
|
||||
f"LORA_CHECKS=0 RETAINED_METRICS={retained}"
|
||||
)
|
||||
assert output.count(marker) == 7, output
|
||||
assert f"'{problem}': 'fail'" in output, output
|
||||
|
||||
Reference in New Issue
Block a user