Support per-regex diff-threshold predicates in the tensor comparator (#30654)

test/registered/debug_utils/comparator/tensor_comparator/test_comparator.py

@@ -12,6 +12,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.comparator import (
     compute_tensor_info,
 )
 from sglang.srt.debug_utils.comparator.tensor_comparator.types import DiffInfo
+from sglang.srt.debug_utils.comparator.threshold_dsl import DiffThresholdRule
 from sglang.test.ci.ci_register import register_cpu_ci
 
 register_cpu_ci(est_time=20, suite="base-a-test-cpu", nightly=True)
@@ -269,9 +270,7 @@ class TestComputeDiff:
         x: torch.Tensor = torch.randn(8, 16)
         y: torch.Tensor = x + torch.randn_like(x) * 0.01
 
-        diff: DiffInfo = compute_diff(
-            x_baseline=x, x_target=y, diff_threshold=1e-3, seq_dim=0
-        )
+        diff: DiffInfo = compute_diff(x_baseline=x, x_target=y, seq_dim=0)
 
         assert diff.per_token_rel_diff is not None
         assert isinstance(diff.per_token_rel_diff, list)
@@ -283,7 +282,7 @@ class TestComputeDiff:
         x: torch.Tensor = torch.randn(8, 16)
         y: torch.Tensor = x + torch.randn_like(x) * 0.01
 
-        diff: DiffInfo = compute_diff(x_baseline=x, x_target=y, diff_threshold=1e-3)
+        diff: DiffInfo = compute_diff(x_baseline=x, x_target=y)
 
         assert diff.per_token_rel_diff is None
 
@@ -293,9 +292,7 @@ class TestComputeDiff:
         x: torch.Tensor = torch.randn(4, 8)
         y: torch.Tensor = x + torch.randn_like(x) * 0.01
 
-        diff: DiffInfo = compute_diff(
-            x_baseline=x, x_target=y, diff_threshold=1e-3, seq_dim=0
-        )
+        diff: DiffInfo = compute_diff(x_baseline=x, x_target=y, seq_dim=0)
 
         json_str: str = diff.model_dump_json()
         assert "per_token_rel_diff" in json_str
@@ -376,5 +373,160 @@ class TestCompareTensors:
         assert info.target.sample is None
 
 
+class TestComputeDiffPredicate:
+    @staticmethod
+    def _near_zero_pair() -> tuple[torch.Tensor, torch.Tensor]:
+        """Sign-flipped near-zero pair: rel_diff == 2.0 but max_abs/mean_abs == 2e-5."""
+        x = torch.tensor([1e-5, -1e-5, 1e-5, -1e-5])
+        return x, -x
+
+    def test_default_predicate(self) -> None:
+        """No predicate → the default 'rel <= 0.001'; near-zero pair fails and the string is recorded."""
+        x, y = self._near_zero_pair()
+        diff = compute_diff(x_baseline=x, x_target=y)
+
+        assert diff.rel_diff == pytest.approx(2.0, abs=1e-4)
+        assert diff.max_abs_diff == pytest.approx(2e-5, abs=1e-7)
+        assert diff.predicate == "rel <= 0.001"
+        assert diff.passed is False
+
+    def test_predicate_rescues_near_zero_via_max_abs(self) -> None:
+        """A 'rel or max_abs' predicate passes the near-zero pair despite a failing rel."""
+        x, y = self._near_zero_pair()
+        diff = compute_diff(
+            x_baseline=x, x_target=y, predicate="rel <= 0.0085 or max_abs <= 1e-4"
+        )
+
+        assert diff.rel_diff > 1.0  # relative term still fails
+        assert diff.passed is True
+        assert diff.predicate == "rel <= 0.0085 or max_abs <= 1e-4"
+
+    def test_predicate_does_not_rescue_real_magnitude_diff(self) -> None:
+        """A real-magnitude diff fails both terms of a 'rel or max_abs' predicate."""
+        x = torch.ones(10)
+        y = x.clone()
+        y[0] = 2.0  # max_abs_diff == 1.0, rel_diff ~0.043
+        diff = compute_diff(
+            x_baseline=x, x_target=y, predicate="rel <= 0.0085 or max_abs <= 1e-3"
+        )
+
+        assert diff.max_abs_diff == pytest.approx(1.0, abs=1e-4)
+        assert diff.passed is False
+
+    def test_and_predicate_requires_both(self) -> None:
+        """'rel and max_abs' fails the near-zero pair (rel huge) but passes a small-both diff."""
+        x, y = self._near_zero_pair()
+        assert (
+            compute_diff(
+                x_baseline=x, x_target=y, predicate="rel <= 0.0085 and max_abs <= 1e-4"
+            ).passed
+            is False
+        )
+        small = torch.ones(10)
+        small_y = small + 1e-4  # rel ~5e-9, max_abs 1e-4
+        assert (
+            compute_diff(
+                x_baseline=small,
+                x_target=small_y,
+                predicate="rel <= 0.0085 and max_abs <= 1e-3",
+            ).passed
+            is True
+        )
+
+    def test_mean_abs_variable(self) -> None:
+        """A mean_abs predicate uses the mean absolute diff (2e-5 for the near-zero pair)."""
+        x, y = self._near_zero_pair()
+        assert (
+            compute_diff(x_baseline=x, x_target=y, predicate="mean_abs <= 1e-4").passed
+            is True
+        )
+        assert (
+            compute_diff(x_baseline=x, x_target=y, predicate="mean_abs <= 1e-6").passed
+            is False
+        )
+
+    def test_boundary_le_inclusive(self) -> None:
+        """<= includes the boundary, < excludes it (max_abs_diff == 0.5 exactly)."""
+        x = torch.tensor([1.0, 1.0])
+        y = torch.tensor([1.5, 1.5])
+        assert (
+            compute_diff(x_baseline=x, x_target=y, predicate="max_abs <= 0.5").passed
+            is True
+        )
+        assert (
+            compute_diff(x_baseline=x, x_target=y, predicate="max_abs < 0.5").passed
+            is False
+        )
+
+    def test_predicate_recorded_for_empty_tensor(self) -> None:
+        """Empty tensors short-circuit to passed=True and still record the predicate."""
+        empty = torch.empty(0)
+        diff = compute_diff(x_baseline=empty, x_target=empty, predicate="rel <= 0")
+
+        assert diff.passed is True
+        assert diff.predicate == "rel <= 0"
+
+    def test_predicate_json_roundtrip(self) -> None:
+        """DiffInfo.predicate survives JSON serialization."""
+        x, y = self._near_zero_pair()
+        diff = compute_diff(
+            x_baseline=x, x_target=y, predicate="rel <= 0.0085 or max_abs <= 1e-4"
+        )
+
+        roundtripped = DiffInfo.model_validate_json(diff.model_dump_json())
+        assert roundtripped.predicate == "rel <= 0.0085 or max_abs <= 1e-4"
+        assert roundtripped.passed is True
+
+
+class TestCompareTensorPairPredicate:
+    def test_predicate_resolved_per_name(self) -> None:
+        """compare_tensor_pair resolves the per-regex predicate by tensor name into the verdict."""
+        x = torch.tensor([1e-5, -1e-5, 1e-5, -1e-5])
+        y = -x
+
+        without = compare_tensor_pair(x_baseline=x, x_target=y, name="g.expert.0")
+        assert without.diff is not None
+        assert without.diff.passed is False
+
+        with_pred = compare_tensor_pair(
+            x_baseline=x,
+            x_target=y,
+            name="g.expert.0",
+            diff_threshold_rules=[
+                DiffThresholdRule(".*expert.*", "rel <= 0.0085 or max_abs <= 1e-4")
+            ],
+        )
+        assert with_pred.diff is not None
+        assert with_pred.diff.passed is True
+        assert with_pred.diff.predicate == "rel <= 0.0085 or max_abs <= 1e-4"
+
+    def test_unmatched_name_raises(self) -> None:
+        """A tensor matching no pattern raises (fail-closed)."""
+        x = torch.tensor([1e-5, -1e-5, 1e-5, -1e-5])
+        with pytest.raises(ValueError, match="matched no --diff-threshold pattern"):
+            compare_tensor_pair(
+                x_baseline=x,
+                x_target=-x,
+                name="g.attn.qkv",
+                diff_threshold_rules=[
+                    DiffThresholdRule(".*expert.*", "rel <= 0.0085 or max_abs <= 1e-4")
+                ],
+            )
+
+    def test_predicate_propagates_to_downcast_diff(self) -> None:
+        """When baseline/target dtypes differ, the resolved predicate also drives the downcast diff."""
+        x = torch.tensor([1e-5, -1e-5, 1e-5, -1e-5])
+        info = compare_tensor_pair(
+            x_baseline=x,
+            x_target=(-x).to(torch.bfloat16),
+            name="g.expert.0",
+            diff_threshold_rules=[
+                DiffThresholdRule(".*expert.*", "rel <= 0.0085 or max_abs <= 1e-4")
+            ],
+        )
+        assert info.diff_downcast is not None
+        assert info.diff_downcast.predicate == "rel <= 0.0085 or max_abs <= 1e-4"
+
+
 if __name__ == "__main__":
     sys.exit(pytest.main([__file__]))

test/registered/debug_utils/comparator/tensor_comparator/test_formatter.py

@@ -1,4 +1,9 @@
 import sys
+from pathlib import Path
+
+_TEST_ROOT: Path = Path(__file__).resolve().parents[4]
+if str(_TEST_ROOT) not in sys.path:
+    sys.path.insert(0, str(_TEST_ROOT))
 
 import pytest
 from registered.debug_utils.comparator.testing_helpers import (
@@ -117,6 +122,19 @@ class TestFormatComparison:
             "[abs_diff] p1=0.0001 p5=0.0001 p50=0.0002 p95=0.0004 p99=0.0005"
         )
 
+    def test_marker_follows_passed_not_rel_threshold(self):
+        """The ✅/❌ marker reflects diff.passed (the predicate verdict), not rel_diff vs a threshold."""
+        info = TensorComparisonInfo(
+            name="rescued",
+            baseline=_make_tensor_info(),
+            target=_make_tensor_info(),
+            unified_shape=[4, 8],
+            shape_mismatch=False,
+            diff=_make_diff(rel_diff=2.0, passed=True),
+        )
+
+        assert "✅ rel_diff=2.0\t" in format_comparison(info)
+
     def test_shape_mismatch(self):
         info = TensorComparisonInfo(
             name="mismatch",

test/registered/debug_utils/comparator/tensor_comparator/test_types.py

@@ -46,7 +46,6 @@ def _make_diff(**overrides) -> DiffInfo:
         max_diff_coord=[2, 3],
         baseline_at_max=1.0,
         target_at_max=1.0005,
-        diff_threshold=1e-3,
         passed=True,
     )
     defaults.update(overrides)
@@ -77,7 +76,6 @@ class TestStrictBase:
                 max_diff_coord=[0],
                 baseline_at_max=0.0,
                 target_at_max=0.0,
-                diff_threshold=1e-3,
                 passed=True,
                 extra_field=123,
             )
@@ -140,7 +138,6 @@ def _make_replicated_check(**overrides) -> ReplicatedCheckResult:
             rel_diff=0.1,
             max_abs_diff=0.1,
             mean_abs_diff=0.05,
-            diff_threshold=1e-6,
             passed=False,
         ),
     )

test/registered/debug_utils/comparator/test_entrypoint.py

@@ -2709,7 +2709,8 @@ def _make_argv(
     preset: str | None = None,
     grouping_skip_keys: list[str] | None = None,
     token_aligner: str | None = None,
-    diff_threshold: float = 1e-3,
+    diff_threshold: float | None = 1e-3,
+    diff_thresholds: list[tuple[str, str]] | None = None,
     output_format: str = "json",
     start_step: int | None = None,
     end_step: int | None = None,
@@ -2730,8 +2731,6 @@ def _make_argv(
         str(baseline_path),
         "--target-path",
         str(target_path),
-        "--diff-threshold",
-        str(diff_threshold),
         "--output-format",
         output_format,
     ]
@@ -2768,6 +2767,12 @@ def _make_argv(
         argv += ["--viz-output-dir", viz_output_dir]
     if visualize_per_token is not None:
         argv += ["--visualize-per-token", visualize_per_token]
+    if diff_thresholds is not None:
+        argv.append("--diff-threshold")
+        for pattern, predicate in diff_thresholds:
+            argv += [pattern, predicate]
+    elif diff_threshold is not None:
+        argv += ["--diff-threshold", str(diff_threshold)]
 
     return argv
 
@@ -4991,5 +4996,250 @@ class TestErrorResilience:
         assert hint_pos < traceback_pos
 
 
+class TestDiffThresholdCliParsing:
+    def test_parse_args_collects_diff_threshold_tokens(self) -> None:
+        """parse_args leaves --diff-threshold as raw tokens (None when the flag is absent)."""
+        argv = ["--baseline-path", "b", "--target-path", "t"]
+        assert parse_args(argv).diff_threshold is None
+
+        argv += ["--diff-threshold", ".*expert.*", "rel <= 0.0085 or max_abs <= 1e-3"]
+        assert parse_args(argv).diff_threshold == [
+            ".*expert.*",
+            "rel <= 0.0085 or max_abs <= 1e-3",
+        ]
+
+
+class TestDiffThresholdPredicateExitCode:
+    """End-to-end: a per-regex --diff-threshold predicate drives the per-tensor verdict and exit code."""
+
+    @staticmethod
+    def _dump_near_zero_pair(tmp_path: Path) -> tuple[Path, Path]:
+        baseline_t = torch.tensor([[1e-5, -1e-5], [1e-5, -1e-5]])
+        baseline = _create_rank_dump(
+            tmp_path / "baseline", rank=0, name="g", tensor=baseline_t
+        )
+        target = _create_rank_dump(
+            tmp_path / "target", rank=0, name="g", tensor=-baseline_t
+        )
+        return baseline, target
+
+    def test_default_predicate_fails_near_zero(self, tmp_path, capsys) -> None:
+        """The default 'rel <= X' predicate fails the near-zero pair (exit code 1)."""
+        baseline, target = self._dump_near_zero_pair(tmp_path)
+        argv = _make_argv(baseline, target, diff_threshold=0.0085)
+
+        records, exit_code = _run_and_parse(argv, capsys)
+
+        tensors = [r for r in records if isinstance(r, ComparisonTensorRecord)]
+        assert len(tensors) == 1
+        assert tensors[0].diff is not None and tensors[0].diff.passed is False
+        assert tensors[0].diff.predicate == "rel <= 0.0085"
+        assert exit_code == 1
+
+    def test_predicate_passes_near_zero(self, tmp_path, capsys) -> None:
+        """A 'rel or max_abs' predicate passes the near-zero pair (exit code 0)."""
+        baseline, target = self._dump_near_zero_pair(tmp_path)
+        argv = _make_argv(
+            baseline,
+            target,
+            diff_thresholds=[(".*", "rel <= 0.0085 or max_abs <= 1e-4")],
+        )
+
+        records, exit_code = _run_and_parse(argv, capsys)
+
+        tensors = [r for r in records if isinstance(r, ComparisonTensorRecord)]
+        assert len(tensors) == 1
+        assert tensors[0].diff is not None and tensors[0].diff.passed is True
+        assert tensors[0].diff.predicate == "rel <= 0.0085 or max_abs <= 1e-4"
+        summary = records[-1]
+        assert isinstance(summary, SummaryRecord)
+        assert summary.failed == 0
+        assert exit_code == 0
+
+    def test_predicate_does_not_rescue_real_magnitude_failure(
+        self, tmp_path, capsys
+    ) -> None:
+        """With a co-located passing tensor, a real-magnitude divergence still fails (exit code 1)."""
+        ones = torch.ones(4, 4)
+        real_target = ones.clone()
+        real_target[0, 0] = 5.0  # max_abs_diff == 4.0, far above the floor
+
+        baseline = _create_rank_dump(
+            tmp_path / "baseline",
+            rank=0,
+            name="g_real",
+            tensor=ones,
+            extra_dumps=[("g_pass", ones)],
+        )
+        target = _create_rank_dump(
+            tmp_path / "target",
+            rank=0,
+            name="g_real",
+            tensor=real_target,
+            extra_dumps=[("g_pass", ones)],
+        )
+        argv = _make_argv(
+            baseline,
+            target,
+            diff_thresholds=[(".*", "rel <= 0.0085 or max_abs <= 1e-3")],
+        )
+
+        records, exit_code = _run_and_parse(argv, capsys)
+
+        by_name = {r.name: r for r in records if isinstance(r, ComparisonTensorRecord)}
+        assert (
+            by_name["g_pass"].diff is not None and by_name["g_pass"].diff.passed is True
+        )
+        assert (
+            by_name["g_real"].diff is not None
+            and by_name["g_real"].diff.passed is False
+        )
+        summary = records[-1]
+        assert isinstance(summary, SummaryRecord)
+        assert summary.passed == 1 and summary.failed == 1
+        assert exit_code == 1
+
+    def test_per_regex_selectivity_with_catch_all(self, tmp_path, capsys) -> None:
+        """A specific predicate rescues only its name; the catch-all keeps the rest strict (exit code 1)."""
+        near_zero = torch.tensor([[1e-5, -1e-5], [1e-5, -1e-5]])
+
+        baseline = _create_rank_dump(
+            tmp_path / "baseline",
+            rank=0,
+            name="g_apple",
+            tensor=near_zero,
+            extra_dumps=[("g_orange", near_zero)],
+        )
+        target = _create_rank_dump(
+            tmp_path / "target",
+            rank=0,
+            name="g_apple",
+            tensor=-near_zero,
+            extra_dumps=[("g_orange", -near_zero)],
+        )
+        argv = _make_argv(
+            baseline,
+            target,
+            diff_thresholds=[
+                (".*apple.*", "rel <= 0.0085 or max_abs <= 1e-4"),
+                (".*", "rel <= 0.0085"),
+            ],
+        )
+
+        records, exit_code = _run_and_parse(argv, capsys)
+
+        by_name = {r.name: r for r in records if isinstance(r, ComparisonTensorRecord)}
+        assert by_name["g_apple"].diff is not None
+        assert by_name["g_apple"].diff.passed is True  # rescued by max_abs
+        assert by_name["g_orange"].diff is not None
+        assert by_name["g_orange"].diff.passed is False  # catch-all strict rel
+        assert exit_code == 1
+
+    def test_unmatched_tensor_errors(self, tmp_path, capsys) -> None:
+        """A tensor matching no pattern (no catch-all) becomes an error -> exit code 1 (fail-closed)."""
+        near_zero = torch.tensor([[1e-5, -1e-5], [1e-5, -1e-5]])
+
+        baseline = _create_rank_dump(
+            tmp_path / "baseline", rank=0, name="g_orange", tensor=near_zero
+        )
+        target = _create_rank_dump(
+            tmp_path / "target", rank=0, name="g_orange", tensor=-near_zero
+        )
+        argv = _make_argv(
+            baseline,
+            target,
+            diff_thresholds=[(".*apple.*", "rel <= 0.0085 or max_abs <= 1e-4")],
+        )
+
+        records, exit_code = _run_and_parse(argv, capsys)
+
+        errors = [r for r in records if isinstance(r, ComparisonErrorRecord)]
+        assert len(errors) == 1
+        assert "matched no --diff-threshold pattern" in errors[0].exception_message
+        assert exit_code == 1
+
+    def test_diff_threshold_flag_omitted_uses_default_predicate(
+        self, tmp_path, capsys
+    ) -> None:
+        """With no --diff-threshold flag, run() falls back to the default 'rel <= 0.001' rule."""
+        baseline, target = self._dump_near_zero_pair(tmp_path)
+        argv = _make_argv(baseline, target, diff_threshold=None)
+        assert "--diff-threshold" not in argv
+
+        records, exit_code = _run_and_parse(argv, capsys)
+
+        tensors = [r for r in records if isinstance(r, ComparisonTensorRecord)]
+        assert len(tensors) == 1
+        assert tensors[0].diff is not None
+        assert tensors[0].diff.predicate == "rel <= 0.001"
+        assert tensors[0].diff.passed is False
+        assert exit_code == 1
+
+    def test_miles_per_regex_selectivity_mixed_verdicts(self, tmp_path, capsys) -> None:
+        """'.*apple.*' tensors get a max_abs rescue, the strict catch-all does not: an identical near-zero pair passes as apple.weight yet fails as banana.bias."""
+        near_zero = torch.tensor([[1e-5, -1e-5], [1e-5, -1e-5]])
+        ones = torch.ones(4, 4)
+        ones_spike = ones.clone()
+        ones_spike[0, 0] = 5.0
+
+        baseline = _create_rank_dump(
+            tmp_path / "baseline",
+            rank=0,
+            name="layer.apple.weight",
+            tensor=near_zero,
+            extra_dumps=[
+                ("layer.apple.bias", ones),
+                ("layer.banana.weight", ones),
+                ("layer.banana.bias", near_zero),
+            ],
+        )
+        target = _create_rank_dump(
+            tmp_path / "target",
+            rank=0,
+            name="layer.apple.weight",
+            tensor=-near_zero,
+            extra_dumps=[
+                ("layer.apple.bias", ones_spike),
+                ("layer.banana.weight", ones + 1e-5),
+                ("layer.banana.bias", -near_zero),
+            ],
+        )
+        argv = _make_argv(
+            baseline,
+            target,
+            diff_thresholds=[
+                (".*apple.*", "rel < 1e-3 or max_abs < 0.01"),
+                (".*", "rel < 1e-3"),
+            ],
+        )
+
+        records, exit_code = _run_and_parse(argv, capsys)
+
+        by_name = {r.name: r for r in records if isinstance(r, ComparisonTensorRecord)}
+        assert {*by_name} == {
+            "layer.apple.weight",
+            "layer.apple.bias",
+            "layer.banana.weight",
+            "layer.banana.bias",
+        }
+        assert all(r.diff is not None for r in by_name.values())
+
+        apple_predicate = "rel < 1e-3 or max_abs < 0.01"
+        assert by_name["layer.apple.weight"].diff.predicate == apple_predicate
+        assert by_name["layer.apple.bias"].diff.predicate == apple_predicate
+        assert by_name["layer.banana.weight"].diff.predicate == "rel < 1e-3"
+        assert by_name["layer.banana.bias"].diff.predicate == "rel < 1e-3"
+
+        assert by_name["layer.apple.weight"].diff.passed is True
+        assert by_name["layer.apple.bias"].diff.passed is False
+        assert by_name["layer.banana.weight"].diff.passed is True
+        assert by_name["layer.banana.bias"].diff.passed is False
+
+        summary = records[-1]
+        assert isinstance(summary, SummaryRecord)
+        assert summary.passed == 2 and summary.failed == 2
+        assert exit_code == 1
+
+
 if __name__ == "__main__":
     sys.exit(pytest.main([__file__]))

test/registered/debug_utils/comparator/test_manually_verify.py

@@ -234,7 +234,6 @@ class TestPerTokenHeatmapManualVerify:
                 x_baseline=baseline,
                 x_target=target,
                 name=f"layer_{i}_hidden_states",
-                diff_threshold=1e-3,
                 seq_dim=0,
             )
             records.append(ComparisonTensorRecord(**info.model_dump()))
@@ -278,7 +277,6 @@ class TestPerTokenHeatmapManualVerify:
                 x_baseline=baseline,
                 x_target=target,
                 name=f"layer_{i}_attn_output",
-                diff_threshold=1e-3,
                 seq_dim=0,
             )
             records.append(ComparisonTensorRecord(**info.model_dump()))

test/registered/debug_utils/comparator/test_model_validation.py

@@ -215,7 +215,6 @@ def _make_diff_info(*, passed: bool) -> DiffInfo:
         max_diff_coord=[0, 0],
         baseline_at_max=1.0,
         target_at_max=1.01,
-        diff_threshold=1e-3,
         passed=passed,
     )
 

test/registered/debug_utils/comparator/test_output_types.py

@@ -1,5 +1,10 @@
 import sys
 from io import StringIO
+from pathlib import Path
+
+_TEST_ROOT: Path = Path(__file__).resolve().parents[3]
+if str(_TEST_ROOT) not in sys.path:
+    sys.path.insert(0, str(_TEST_ROOT))
 
 import pytest
 from registered.debug_utils.comparator.testing_helpers import (

test/registered/debug_utils/comparator/test_per_token_visualizer.py

@@ -37,7 +37,6 @@ def _make_comparison_record(
         x_baseline=baseline,
         x_target=target,
         name=name,
-        diff_threshold=1e-3,
         seq_dim=seq_dim,
     )
     return ComparisonTensorRecord(**info.model_dump())
@@ -66,7 +65,6 @@ class TestPerTokenVisualizer:
             x_baseline=torch.randn(4, 8),
             x_target=torch.randn(4, 8),
             name="no_per_token",
-            diff_threshold=1e-3,
         )
         record = ComparisonTensorRecord(**info.model_dump())
 

test/registered/debug_utils/comparator/test_threshold_dsl.py

@@ -0,0 +1,221 @@
+import sys
+
+import pytest
+
+from sglang.srt.debug_utils.comparator.threshold_dsl import (
+    DiffThresholdRule,
+    evaluate_predicate,
+    parse_diff_threshold_rules,
+    parse_predicate,
+    resolve_predicate,
+)
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+
+
+def _ev(
+    expr: str, *, rel: float = 0.0, max_abs: float = 0.0, mean_abs: float = 0.0
+) -> bool:
+    return evaluate_predicate(
+        parse_predicate(expr), rel=rel, max_abs=max_abs, mean_abs=mean_abs
+    )
+
+
+class TestParsePredicate:
+    @pytest.mark.parametrize(
+        "expr",
+        [
+            "rel <= 0.0085",
+            "rel < 1",
+            "max_abs > 0",
+            "mean_abs >= 1e-5",
+            "rel <= 0.01 or max_abs <= 1e-4",
+            "rel <= 0.01 and max_abs <= 1e-4",
+            "(rel <= 0.01 and max_abs <= 1e-4) or mean_abs <= 1e-5",
+            "0 <= rel < 1",
+            "rel <= -0.0",
+            "rel <= 0",
+        ],
+    )
+    def test_valid_predicates_parse(self, expr: str) -> None:
+        """All supported forms parse without error."""
+        parse_predicate(expr)
+
+    @pytest.mark.parametrize(
+        "expr",
+        [
+            "abs(rel) < 1",
+            "rel.x < 1",
+            "foo < 1",
+            "rel < 'x'",
+            "",
+            "rel <",
+        ],
+    )
+    def test_invalid_predicates_raise(self, expr: str) -> None:
+        """Unknown names, attribute access, bad types, and syntax errors raise ValueError."""
+        with pytest.raises(ValueError):
+            parse_predicate(expr)
+
+    def test_unknown_name_message_lists_allowed(self) -> None:
+        """The error for an unknown variable names the allowed variables."""
+        with pytest.raises(ValueError, match="rel.*max_abs.*mean_abs"):
+            parse_predicate("foo < 1")
+
+
+class TestEvaluatePredicate:
+    def test_rel_only_true_and_false(self) -> None:
+        """A pure rel predicate uses only rel."""
+        assert _ev("rel <= 0.01", rel=0.005) is True
+        assert _ev("rel <= 0.01", rel=0.02) is False
+
+    def test_le_boundary_inclusive(self) -> None:
+        """<= includes the boundary; < excludes it."""
+        assert _ev("rel <= 0.01", rel=0.01) is True
+        assert _ev("rel < 0.01", rel=0.01) is False
+
+    def test_or_short_circuit_semantics(self) -> None:
+        """or passes if either side holds (near-zero rescue pattern)."""
+        assert _ev("rel <= 0.0085 or max_abs <= 1e-3", rel=2.0, max_abs=2e-5) is True
+        assert _ev("rel <= 0.0085 or max_abs <= 1e-3", rel=2.0, max_abs=0.5) is False
+
+    def test_and_requires_both(self) -> None:
+        """and passes only if both sides hold."""
+        assert _ev("rel <= 0.01 and max_abs <= 1e-3", rel=0.005, max_abs=1e-4) is True
+        assert _ev("rel <= 0.01 and max_abs <= 1e-3", rel=0.005, max_abs=0.5) is False
+
+    def test_mean_abs_variable(self) -> None:
+        """mean_abs is a usable variable."""
+        assert _ev("mean_abs <= 1e-5", mean_abs=1e-6) is True
+        assert _ev("mean_abs <= 1e-5", mean_abs=1e-4) is False
+
+    def test_parentheses_grouping(self) -> None:
+        """Parentheses override and/or precedence."""
+        assert (
+            _ev(
+                "(rel <= 0.01 and max_abs <= 1e-4) or mean_abs <= 1e-5",
+                rel=2.0,
+                max_abs=2.0,
+                mean_abs=1e-6,
+            )
+            is True
+        )
+
+    def test_chained_comparison(self) -> None:
+        """Chained comparison follows Python all-must-hold semantics."""
+        assert _ev("0 <= rel < 1", rel=0.5) is True
+        assert _ev("0 <= rel < 1", rel=1.5) is False
+
+    def test_bitwise_zero_predicate(self) -> None:
+        """rel <= 0 passes only for an exactly-zero rel (bitwise)."""
+        assert _ev("rel <= 0", rel=0.0) is True
+        assert _ev("rel <= 0", rel=1e-12) is False
+
+
+class TestDiffThresholdRule:
+    def test_is_frozen_value_object(self) -> None:
+        """DiffThresholdRule is a frozen, value-equal dataclass."""
+        assert DiffThresholdRule(".*", "rel <= 1e-3") == DiffThresholdRule(
+            ".*", "rel <= 1e-3"
+        )
+        with pytest.raises(Exception):
+            DiffThresholdRule(".*", "rel <= 1e-3").pattern = "x"
+
+
+class TestParseDiffThresholdRules:
+    def test_none_and_empty_return_default(self) -> None:
+        """Missing flag (None) and bare flag (empty list) both yield the caller's default."""
+        assert parse_diff_threshold_rules(None, default_predicate="rel <= 0.001") == [
+            DiffThresholdRule(".*", "rel <= 0.001")
+        ]
+        assert parse_diff_threshold_rules([], default_predicate="rel <= 0.001") == [
+            DiffThresholdRule(".*", "rel <= 0.001")
+        ]
+
+    def test_single_float_shorthand(self) -> None:
+        """A single float token expands to a global 'rel <= X' rule."""
+        assert parse_diff_threshold_rules(
+            ["0.0085"], default_predicate="rel <= 0.001"
+        ) == [DiffThresholdRule(".*", "rel <= 0.0085")]
+
+    def test_single_non_float_raises(self) -> None:
+        """A single non-float token is a usage error (not a valid shorthand)."""
+        with pytest.raises(ValueError):
+            parse_diff_threshold_rules([".*expert.*"], default_predicate="rel <= 0.001")
+
+    def test_pairs_parsed_in_order(self) -> None:
+        """Flat regex/predicate tokens parse into ordered rules."""
+        assert parse_diff_threshold_rules(
+            [".*apple.*", "rel <= 0.01 or max_abs <= 1e-4", ".*", "rel <= 0.0085"],
+            default_predicate="rel <= 0.001",
+        ) == [
+            DiffThresholdRule(".*apple.*", "rel <= 0.01 or max_abs <= 1e-4"),
+            DiffThresholdRule(".*", "rel <= 0.0085"),
+        ]
+
+    def test_odd_number_of_tokens_raises(self) -> None:
+        """An unpaired token is a usage error."""
+        with pytest.raises(ValueError):
+            parse_diff_threshold_rules(
+                [".*apple.*", "rel <= 0.01", ".*orange.*"],
+                default_predicate="rel <= 0.001",
+            )
+
+    def test_bad_predicate_raises(self) -> None:
+        """A malformed predicate fails fast at parse time."""
+        with pytest.raises(ValueError):
+            parse_diff_threshold_rules(
+                [".*", "rel <= "], default_predicate="rel <= 0.001"
+            )
+
+
+class TestResolvePredicate:
+    def test_none_or_empty_returns_explicit_default(self) -> None:
+        """No rules → the supplied default predicate."""
+        assert (
+            resolve_predicate("x", None, default_predicate="rel <= 1e-3")
+            == "rel <= 1e-3"
+        )
+        assert (
+            resolve_predicate("x", [], default_predicate="rel <= 1e-3") == "rel <= 1e-3"
+        )
+
+    def test_first_matching_pattern_wins(self) -> None:
+        """Rules are tried in order; the first fullmatch wins (specific before general)."""
+        rules = [DiffThresholdRule(".*expert.*", "P1"), DiffThresholdRule(".*", "P2")]
+        assert (
+            resolve_predicate("layer.expert.weight", rules, default_predicate="D")
+            == "P1"
+        )
+        assert (
+            resolve_predicate("layer.attn.weight", rules, default_predicate="D") == "P2"
+        )
+
+    def test_unmatched_name_raises(self) -> None:
+        """A name matching no pattern is a fail-closed error."""
+        with pytest.raises(ValueError, match="matched no --diff-threshold pattern"):
+            resolve_predicate(
+                "k_layernorm",
+                [DiffThresholdRule(".*expert.*", "P")],
+                default_predicate="D",
+            )
+
+    def test_fullmatch_semantics(self) -> None:
+        """Matching is fullmatch: a partial pattern does not match a longer name (→ raises)."""
+        assert (
+            resolve_predicate(
+                "expert", [DiffThresholdRule("expert", "P")], default_predicate="D"
+            )
+            == "P"
+        )
+        with pytest.raises(ValueError):
+            resolve_predicate(
+                "layer.expert.weight",
+                [DiffThresholdRule("expert", "P")],
+                default_predicate="D",
+            )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__]))

test/registered/debug_utils/comparator/testing_helpers.py

@@ -63,7 +63,6 @@ def make_diff(
     max_abs_diff: float = 0.0005,
     mean_abs_diff: float = 0.0002,
     abs_diff_percentiles: Optional[dict[int, float]] = None,
-    diff_threshold: float = 1e-3,
     passed: bool = True,
 ) -> DiffInfo:
     return DiffInfo(
@@ -78,7 +77,6 @@ def make_diff(
         max_diff_coord=[2, 3],
         baseline_at_max=1.0,
         target_at_max=1.0005,
-        diff_threshold=diff_threshold,
         passed=passed,
     )