From 3420baea8c7a4812b6a35aebd33d9946fad475f7 Mon Sep 17 00:00:00 2001 From: Mick Date: Fri, 16 Oct 2026 11:04:05 +0800 Subject: [PATCH] [diffusion] CI: batch performance failure-policy runner scenarios (#44508) --- .../unit/test_performance_failure_policy.py | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py b/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py index b5b7d438aa..9d79e0894e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py +++ b/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py @@ -197,9 +197,9 @@ def test_retry_estimate_excludes_successful_cases(tmp_path): assert _estimate_failed_test_time(None, 130) == 130 -@pytest.mark.parametrize( - "problem", - [ +def test_performance_failure_survives_real_pytest_runner(tmp_path): + # share imports across failure scenarios while exercising all seven real attempts + problems = [ "regression", "missing_baseline", "missing_record", @@ -209,10 +209,7 @@ def test_retry_estimate_excludes_successful_cases(tmp_path): "e2e_only_missing_baseline", "e2e_only_zero_baseline", "e2e_only_nan_baseline", - ], -) -def test_performance_failure_survives_real_pytest_runner(tmp_path, problem): - # exercise the validator, request loop, pytest output and retry classifier together + ] test_file = tmp_path / "test_guard.py" test_file.write_text( textwrap.dedent( @@ -226,7 +223,7 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem): DiffusionSamplingParams, DiffusionServerArgs, DiffusionTestCase, ScenarioConfig, ) - @pytest.mark.parametrize("case_id", ["threshold_guard"]) + @pytest.mark.parametrize("case_id", PROBLEMS) def test_guard(case_id, monkeypatch, tmp_path): server = common.DiffusionServerBase() server._perf_results = [] @@ -236,14 +233,14 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem): DiffusionSamplingParams(prompt="test"), run_lora_basic_api_check=True, perf_repeat_requests=2, run_consistency_check=False, run_models_api_check=False, - run_perf_check=not PROBLEM.startswith("e2e_only_") and PROBLEM not in ("missing_record", "missing_e2e", "missing_log"), + run_perf_check=not case_id.startswith("e2e_only_") and case_id not in ("missing_record", "missing_e2e", "missing_log"), ) scenario = ScenarioConfig({}, {}, 1000, 100, 100, expected_load_ms=100) - if PROBLEM == "e2e_only_zero_baseline": + if case_id == "e2e_only_zero_baseline": scenario.expected_e2e_ms = 0 - if PROBLEM == "e2e_only_nan_baseline": + if case_id == "e2e_only_nan_baseline": scenario.expected_e2e_ms = float("nan") - if PROBLEM in ("missing_baseline", "e2e_only_missing_baseline"): + if case_id in ("missing_baseline", "e2e_only_missing_baseline"): monkeypatch.delitem(common.BASELINE_CONFIG.scenarios, case_id, raising=False) else: monkeypatch.setitem(common.BASELINE_CONFIG.scenarios, case_id, scenario) @@ -254,15 +251,15 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem): lora_checks = [] def collect(*args, **kwargs): requests.append(1) - if PROBLEM == "missing_record": + if case_id == "missing_record": return None, b"" return RequestPerfRecord( request_id="guard", commit_hash="test", tag="guard", stages=[], steps=[100], - total_duration_ms=None if PROBLEM == "missing_e2e" else 2000, + total_duration_ms=None if case_id == "missing_e2e" else 2000, ), b"" context = SimpleNamespace(load_time_ms=100) - if PROBLEM == "missing_log": + if case_id == "missing_log": log_path = tmp_path / "empty-perf.jsonl" log_path.write_text("") context = SimpleNamespace(perf_log_path=log_path, load_time_ms=100) @@ -284,13 +281,15 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem): try: server._test_diffusion_generation_impl(case, context) finally: - print(f"GUARD_REQUESTS={len(requests)} LORA_CHECKS={len(lora_checks)}") - print(f"RETAINED_METRICS={len(server._perf_results)}") + print( + f"GUARD_CASE={case_id} GUARD_REQUESTS={len(requests)} " + f"LORA_CHECKS={len(lora_checks)} RETAINED_METRICS={len(server._perf_results)}" + ) def test_unrelated_timeout(): raise TimeoutError("independent infrastructure failure") """ - ).replace("PROBLEM", repr(problem)) + ).replace("PROBLEMS", repr(problems)) ) report = tmp_path / "junit.xml" env = os.environ.copy() @@ -315,9 +314,15 @@ def test_performance_failure_survives_real_pytest_runner(tmp_path, problem): output = result.stdout + result.stderr assert result.returncode == 1, output assert "[performance]" in output, output - assert "GUARD_REQUESTS=1 LORA_CHECKS=0" in output, output - retained = 0 if problem in {"missing_record", "missing_e2e", "missing_log"} else 1 - assert f"RETAINED_METRICS={retained}" in output, output assert output.count("Starting pytest attempt") == 7, output assert "Max retry exceeded (6)" in output, output - assert "'threshold_guard': 'fail'" in output, output + for problem in problems: + retained = ( + 0 if problem in {"missing_record", "missing_e2e", "missing_log"} else 1 + ) + marker = ( + f"GUARD_CASE={problem} GUARD_REQUESTS=1 " + f"LORA_CHECKS=0 RETAINED_METRICS={retained}" + ) + assert output.count(marker) == 7, output + assert f"'{problem}': 'fail'" in output, output