Add a debug toggle for selectively reverting PR fixes (#26802)

python/sglang/srt/debug_utils/pr_fix_toggle.py

@@ -0,0 +1,86 @@
+"""Reverse-apply historical PR fixes for regression-style tests."""
+
+from __future__ import annotations
+
+from typing import Dict
+
+from sglang.srt.debug_utils.source_patcher import apply_patches_from_config
+from sglang.srt.environ import envs
+
+_PR_REVERT_YAML_25015 = """
+patches:
+  - target: sglang.srt.speculative.eagle_worker.EAGLEWorker.draft_forward
+    edits:
+      - match: |
+          forward_batch.out_cache_loc = out_cache_loc[i]
+          spec_info.hidden_states = hidden_states
+        replacement: |
+          forward_batch.out_cache_loc = out_cache_loc[i]
+          forward_batch.positions.add_(1)
+          spec_info.hidden_states = hidden_states
+      - match: |
+          hidden_states = logits_output.hidden_states
+          maybe_detect_nan(hidden_states, f"draft_forward step {i}: hidden_states")
+          maybe_detect_inf(hidden_states, f"draft_forward step {i}: hidden_states")
+          forward_batch.positions.add_(1)
+        replacement: |
+          hidden_states = logits_output.hidden_states
+          maybe_detect_nan(hidden_states, f"draft_forward step {i}: hidden_states")
+          maybe_detect_inf(hidden_states, f"draft_forward step {i}: hidden_states")
+
+  - target: sglang.srt.speculative.eagle_worker_v2.EagleDraftWorker.draft_forward
+    edits:
+      - match: |
+          forward_batch.out_cache_loc = out_cache_loc[i]
+          spec_info.hidden_states = hidden_states
+        replacement: |
+          forward_batch.out_cache_loc = out_cache_loc[i]
+          forward_batch.positions.add_(1)
+          spec_info.hidden_states = hidden_states
+      - match: |
+          hidden_states = logits_output.hidden_states
+          forward_batch.positions.add_(1)
+        replacement: |
+          hidden_states = logits_output.hidden_states
+
+  - target: sglang.srt.speculative.eagle_draft_cuda_graph_runner.EAGLEDraftCudaGraphRunner.capture_one_batch_size
+    edits:
+      - match: |
+          forward_batch.spec_info.hidden_states = hidden_states_backup
+          forward_batch.positions.sub_(self.eagle_worker.speculative_num_steps - 1)
+          return ret
+        replacement: |
+          forward_batch.spec_info.hidden_states = hidden_states_backup
+          return ret
+"""
+
+
+_PR_REVERT_YAML_26329 = """
+patches:
+  - target: sglang.srt.speculative.eagle_utils._eagle_prefill_tail_tokens
+    edits:
+      - match: |
+          tail_tokens = next_token_ids.to(batch.input_ids.dtype)
+        prepend: |
+          return next_token_ids.to(batch.input_ids.dtype)
+"""
+
+
+_PR_FIX_REVERT_YAML: Dict[int, str] = {
+    25015: _PR_REVERT_YAML_25015,
+    26329: _PR_REVERT_YAML_26329,
+}
+
+
+def maybe_revert_pr_fix() -> None:
+    if pr_num := envs.SGLANG_DEBUG_REVERT_PR.get():
+        _revert_pr_fix(pr_num)
+
+
+def _revert_pr_fix(pr_num: int) -> None:
+    if pr_num not in _PR_FIX_REVERT_YAML:
+        raise NotImplementedError(
+            f"PR #{pr_num} revert is not registered; "
+            f"available: {sorted(_PR_FIX_REVERT_YAML.keys())}"
+        )
+    apply_patches_from_config(_PR_FIX_REVERT_YAML[pr_num])

python/sglang/srt/debug_utils/source_patcher/source_editor.py

@@ -67,8 +67,9 @@ def _find_match(*, source_lines: list[str], match_lines: list[str]) -> int:
     ]
 
     if len(found_indices) == 0:
-        preview: str = "\n".join(match_lines)
-        raise PatchApplicationError(f"match text not found in source:\n{preview}")
+        raise PatchApplicationError(
+            _not_found_diagnostic(stripped_source, stripped_match)
+        )
     if len(found_indices) > 1:
         preview = "\n".join(match_lines)
         raise PatchApplicationError(
@@ -78,6 +79,41 @@ def _find_match(*, source_lines: list[str], match_lines: list[str]) -> int:
     return found_indices[0]
 
 
+def _not_found_diagnostic(stripped_source: list[str], stripped_match: list[str]) -> str:
+    preview = "\n".join(stripped_match)
+    lines = [
+        f"match text not found in source:\n{preview}",
+        "",
+        f"source_len={len(stripped_source)} lines",
+    ]
+
+    if not stripped_match:
+        return "\n".join(lines)
+    first_match_line = stripped_match[0]
+    hits = [i for i, line in enumerate(stripped_source) if line == first_match_line]
+    if not hits:
+        lines.append(
+            f"first match line {first_match_line!r} does NOT appear anywhere in source"
+        )
+        return "\n".join(lines)
+
+    lines.append(
+        f"first match line {first_match_line!r} appears {len(hits)} time(s); showing up to 8 windows with context:"
+    )
+    for i in hits[:8]:
+        lo = max(0, i - 2)
+        hi = min(len(stripped_source), i + len(stripped_match) + 2)
+        block: list[str] = []
+        for j in range(lo, hi):
+            marker = (
+                ">" if lo + (j - lo) >= i and (j - i) < len(stripped_match) else " "
+            )
+            block.append(f"{marker} {j:4d}: {stripped_source[j]}")
+        lines.append("--")
+        lines.extend(block)
+    return "\n".join(lines)
+
+
 def _realign_replacement(
     *, replacement_lines: list[str], original_indent: int
 ) -> list[str]:

python/sglang/srt/environ.py

@@ -237,6 +237,7 @@ class Envs:
     SGLANG_RECORD_STEP_TIME = EnvBool(False)
     SGLANG_FORCE_SHUTDOWN = EnvBool(False)
     SGLANG_DEBUG_MEMORY_POOL = EnvBool(False)
+    SGLANG_DEBUG_REVERT_PR = EnvInt(0)
     SGLANG_TEST_REQUEST_TIME_STATS = EnvBool(False)
     SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(False)
     SGLANG_SIMULATE_ACC_LEN = EnvFloat(-1)

python/sglang/srt/managers/scheduler.py

@@ -41,6 +41,7 @@ from torch.distributed import barrier
 from sglang.jit_kernel.ngram_embedding import update_token_table
 from sglang.srt.configs.model_config import ModelConfig, ModelImpl
 from sglang.srt.constrained.grammar_manager import GrammarManager
+from sglang.srt.debug_utils.pr_fix_toggle import maybe_revert_pr_fix
 from sglang.srt.disaggregation.decode import (
     DecodePreallocQueue,
     DecodeTransferQueue,
@@ -553,6 +554,8 @@ class Scheduler(
 
         self.init_batch_result_processor()
 
+        maybe_revert_pr_fix()
+
         self.is_initializing = False
 
     def init_zbal_on_npu(self):

test/registered/debug_utils/source_patcher/test_source_editor.py

@@ -114,6 +114,124 @@ class TestApplyEdits:
         with pytest.raises(PatchApplicationError, match="not found"):
             apply_edits(source=source, edits=edits)
 
+    def test_not_found_diagnostic_reports_source_len(self) -> None:
+        """diagnostic includes total source line count."""
+        source = "line0\nline1\nline2\nline3\nline4\n"
+        edits = [EditSpec(match="absent()", replacement="x")]
+        with pytest.raises(PatchApplicationError) as exc_info:
+            apply_edits(source=source, edits=edits)
+        assert "source_len=5 lines" in str(exc_info.value)
+
+    def test_not_found_diagnostic_when_first_match_line_absent(self) -> None:
+        """diagnostic says 'does NOT appear anywhere' when first line is never present."""
+        source = "def foo():\n    return 1\n"
+        edits = [EditSpec(match="nope_xyz()", replacement="x")]
+        with pytest.raises(PatchApplicationError) as exc_info:
+            apply_edits(source=source, edits=edits)
+        msg = str(exc_info.value)
+        assert "does NOT appear anywhere in source" in msg
+        assert "'nope_xyz()'" in msg
+
+    def test_not_found_diagnostic_single_window_with_marker(self) -> None:
+        """first line is present once but full match doesn't fit: one window with '>' on the match-region line."""
+        source = (
+            "line0\n"
+            "line1\n"
+            "line2\n"
+            "anchor()\n"
+            "wrong_next()\n"
+            "line5\n"
+            "line6\n"
+        )
+        edits = [EditSpec(match="anchor()\nright_next()", replacement="x")]
+        with pytest.raises(PatchApplicationError) as exc_info:
+            apply_edits(source=source, edits=edits)
+        msg = str(exc_info.value)
+        assert "appears 1 time(s)" in msg
+        assert msg.count("--") == 1
+        assert ">    3: anchor()" in msg
+        assert ">    4: wrong_next()" in msg
+        assert "     1: line1" in msg
+        assert "     2: line2" in msg
+        assert "     5: line5" in msg
+        assert "     6: line6" in msg
+
+    def test_not_found_diagnostic_multiple_windows_separated(self) -> None:
+        """when first line appears N (<=8) times, N windows are shown separated by '--'."""
+        source = (
+            "anchor()\n"
+            "tail_a()\n"
+            "filler\n"
+            "anchor()\n"
+            "tail_b()\n"
+            "filler\n"
+            "anchor()\n"
+            "tail_c()\n"
+        )
+        edits = [EditSpec(match="anchor()\nnope()", replacement="x")]
+        with pytest.raises(PatchApplicationError) as exc_info:
+            apply_edits(source=source, edits=edits)
+        msg = str(exc_info.value)
+        assert "appears 3 time(s)" in msg
+        assert msg.count("--") == 3
+        assert ">    0: anchor()" in msg
+        assert ">    3: anchor()" in msg
+        assert ">    6: anchor()" in msg
+
+    def test_not_found_diagnostic_caps_at_8_windows(self) -> None:
+        """when first line appears >8 times, only the first 8 windows are rendered."""
+        source = "\n".join(["anchor()"] * 12) + "\n"
+        edits = [EditSpec(match="anchor()\nnope()", replacement="x")]
+        with pytest.raises(PatchApplicationError) as exc_info:
+            apply_edits(source=source, edits=edits)
+        msg = str(exc_info.value)
+        assert "appears 12 time(s)" in msg
+        assert "up to 8 windows" in msg
+        assert msg.count("--") == 8
+
+    def test_not_found_diagnostic_window_clamps_at_source_boundaries(self) -> None:
+        """window does not include negative indices or indices past the end of source."""
+        source = "anchor()\nfoo\n"
+        edits = [EditSpec(match="anchor()\nbar", replacement="x")]
+        with pytest.raises(PatchApplicationError) as exc_info:
+            apply_edits(source=source, edits=edits)
+        msg = str(exc_info.value)
+        assert ">    0: anchor()" in msg
+        assert ">    1: foo" in msg
+        assert "-1:" not in msg
+        assert "   2:" not in msg
+
+    def test_not_found_diagnostic_multiline_match_marks_full_region(self) -> None:
+        """match spanning N lines: marker '>' covers all N lines of the intended match region."""
+        source = (
+            "filler0\n"
+            "filler1\n"
+            "filler2\n"
+            "filler3\n"
+            "anchor()\n"
+            "middle()\n"
+            "wrong_tail()\n"
+            "filler7\n"
+            "filler8\n"
+            "filler9\n"
+        )
+        edits = [
+            EditSpec(match="anchor()\nmiddle()\nright_tail()", replacement="x"),
+        ]
+        with pytest.raises(PatchApplicationError) as exc_info:
+            apply_edits(source=source, edits=edits)
+        msg = str(exc_info.value)
+        assert ">    4: anchor()" in msg
+        assert ">    5: middle()" in msg
+        assert ">    6: wrong_tail()" in msg
+        assert "     2: filler2" in msg
+        assert "     3: filler3" in msg
+        assert "     7: filler7" in msg
+        assert "     8: filler8" in msg
+        assert "filler0" not in msg
+        assert "filler1" not in msg
+        assert "filler9" not in msg
+
     def test_match_found_multiple_times_raises(self) -> None:
         source = "def foo():\n" "    print(1)\n" "    print(1)\n"
         edits = [EditSpec(match="print(1)", replacement="print(2)")]