[diffusion] CI: batch performance failure-policy runner scenarios (#44508)

This commit is contained in:
Mick
2026-10-16 11:04:05 +08:00
committed by GitHub
parent 6231a617d3
commit 3420baea8c
@@ -197,9 +197,9 @@ def test_retry_estimate_excludes_successful_cases(tmp_path):
assert _estimate_failed_test_time(None, 130) == 130
@pytest.mark.parametrize(
"problem",
[
def test_performance_failure_survives_real_pytest_runner(tmp_path):
# share imports across failure scenarios while exercising all seven real attempts
problems = [
"regression",
"missing_baseline",
"missing_record",
@@ -209,10 +209,7 @@ def test_retry_estimate_excludes_successful_cases(tmp_path):
"e2e_only_missing_baseline",
"e2e_only_zero_baseline",
"e2e_only_nan_baseline",
],
)
def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
# exercise the validator, request loop, pytest output and retry classifier together
]
test_file = tmp_path / "test_guard.py"
test_file.write_text(
textwrap.dedent(
@@ -226,7 +223,7 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
DiffusionSamplingParams, DiffusionServerArgs, DiffusionTestCase, ScenarioConfig,
)
@pytest.mark.parametrize("case_id", ["threshold_guard"])
@pytest.mark.parametrize("case_id", PROBLEMS)
def test_guard(case_id, monkeypatch, tmp_path):
server = common.DiffusionServerBase()
server._perf_results = []
@@ -236,14 +233,14 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
DiffusionSamplingParams(prompt="test"),
run_lora_basic_api_check=True, perf_repeat_requests=2,
run_consistency_check=False, run_models_api_check=False,
run_perf_check=not PROBLEM.startswith("e2e_only_") and PROBLEM not in ("missing_record", "missing_e2e", "missing_log"),
run_perf_check=not case_id.startswith("e2e_only_") and case_id not in ("missing_record", "missing_e2e", "missing_log"),
)
scenario = ScenarioConfig({}, {}, 1000, 100, 100, expected_load_ms=100)
if PROBLEM == "e2e_only_zero_baseline":
if case_id == "e2e_only_zero_baseline":
scenario.expected_e2e_ms = 0
if PROBLEM == "e2e_only_nan_baseline":
if case_id == "e2e_only_nan_baseline":
scenario.expected_e2e_ms = float("nan")
if PROBLEM in ("missing_baseline", "e2e_only_missing_baseline"):
if case_id in ("missing_baseline", "e2e_only_missing_baseline"):
monkeypatch.delitem(common.BASELINE_CONFIG.scenarios, case_id, raising=False)
else:
monkeypatch.setitem(common.BASELINE_CONFIG.scenarios, case_id, scenario)
@@ -254,15 +251,15 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
lora_checks = []
def collect(*args, **kwargs):
requests.append(1)
if PROBLEM == "missing_record":
if case_id == "missing_record":
return None, b""
return RequestPerfRecord(
request_id="guard", commit_hash="test", tag="guard",
stages=[], steps=[100],
total_duration_ms=None if PROBLEM == "missing_e2e" else 2000,
total_duration_ms=None if case_id == "missing_e2e" else 2000,
), b""
context = SimpleNamespace(load_time_ms=100)
if PROBLEM == "missing_log":
if case_id == "missing_log":
log_path = tmp_path / "empty-perf.jsonl"
log_path.write_text("")
context = SimpleNamespace(perf_log_path=log_path, load_time_ms=100)
@@ -284,13 +281,15 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
try:
server._test_diffusion_generation_impl(case, context)
finally:
print(f"GUARD_REQUESTS={len(requests)} LORA_CHECKS={len(lora_checks)}")
print(f"RETAINED_METRICS={len(server._perf_results)}")
print(
f"GUARD_CASE={case_id} GUARD_REQUESTS={len(requests)} "
f"LORA_CHECKS={len(lora_checks)} RETAINED_METRICS={len(server._perf_results)}"
)
def test_unrelated_timeout():
raise TimeoutError("independent infrastructure failure")
"""
).replace("PROBLEM", repr(problem))
).replace("PROBLEMS", repr(problems))
)
report = tmp_path / "junit.xml"
env = os.environ.copy()
@@ -315,9 +314,15 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem):
output = result.stdout + result.stderr
assert result.returncode == 1, output
assert "[performance]" in output, output
assert "GUARD_REQUESTS=1 LORA_CHECKS=0" in output, output
retained = 0 if problem in {"missing_record", "missing_e2e", "missing_log"} else 1
assert f"RETAINED_METRICS={retained}" in output, output
assert output.count("Starting pytest attempt") == 7, output
assert "Max retry exceeded (6)" in output, output
assert "'threshold_guard': 'fail'" in output, output
for problem in problems:
retained = (
0 if problem in {"missing_record", "missing_e2e", "missing_log"} else 1
)
marker = (
f"GUARD_CASE={problem} GUARD_REQUESTS=1 "
f"LORA_CHECKS=0 RETAINED_METRICS={retained}"
)
assert output.count(marker) == 7, output
assert f"'{problem}': 'fail'" in output, output