From 89e24e7a20676eecf91d794e49f80c018ed13956 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sat, 10 Oct 2026 23:52:17 -0700 Subject: [PATCH] [KV canary] Expect sweep counters on the target runner only (#43286) --- .../srt/kv_canary/runner/canary_manager.py | 1 + .../srt/kv_canary/runner/health_checker.py | 6 +++- .../sglang/test/kv_canary/runner_test_base.py | 2 ++ .../kv_canary/test_self_unit_runner_health.py | 31 +++++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/kv_canary/runner/canary_manager.py b/python/sglang/srt/kv_canary/runner/canary_manager.py index 79f4527e68..2603caacae 100644 --- a/python/sglang/srt/kv_canary/runner/canary_manager.py +++ b/python/sglang/srt/kv_canary/runner/canary_manager.py @@ -134,6 +134,7 @@ class CanaryManager: ) self._health_checker = KernelRunCounterHealthChecker( config=config, + is_draft_runner=is_eagle_draft_decode, device_state=self._device_state, active_tags=self._active_tags, outer_step_counter_getter=self._get_outer_step_counter, diff --git a/python/sglang/srt/kv_canary/runner/health_checker.py b/python/sglang/srt/kv_canary/runner/health_checker.py index 62a2b5e235..f8e7c8f31d 100644 --- a/python/sglang/srt/kv_canary/runner/health_checker.py +++ b/python/sglang/srt/kv_canary/runner/health_checker.py @@ -31,12 +31,14 @@ class KernelRunCounterHealthChecker: self, *, config: CanaryConfig, + is_draft_runner: bool, device_state: CanaryDeviceState, active_tags: tuple[CanaryLaunchTag, ...], outer_step_counter_getter: Callable[[], int], d2h_stream: torch.Stream, ) -> None: self._config = config + self._is_draft_runner = is_draft_runner self._device_state = device_state self._active_tags = active_tags self._outer_step_counter_getter = outer_step_counter_getter @@ -76,6 +78,8 @@ class KernelRunCounterHealthChecker: def _expected_active_tags_for_health_check(self) -> tuple[CanaryLaunchTag, ...]: tags = self._active_tags - if self._config.sweep_interval <= 0: + # The scheduler attaches the radix cache only to the target runner, so + # only its sweep launches; a missing attach there must still fail here. + if self._config.sweep_interval <= 0 or self._is_draft_runner: tags = tuple(tag for tag in tags if tag not in _SWEEP_TAGS) return tuple(tag for tag in tags if passes_v_half_gate(tag)) diff --git a/python/sglang/test/kv_canary/runner_test_base.py b/python/sglang/test/kv_canary/runner_test_base.py index b83ca30987..4059cac8a5 100644 --- a/python/sglang/test/kv_canary/runner_test_base.py +++ b/python/sglang/test/kv_canary/runner_test_base.py @@ -74,6 +74,7 @@ def make_manager( req_pool: SimpleNamespace | None = None, per_forward_verify_capacity: int = 16, speculative_num_steps: int = 1, + is_eagle_draft_decode: bool = False, ) -> CanaryManager: if config is None: config = make_config() @@ -95,6 +96,7 @@ def make_manager( per_forward_write_entry_capacity=8, ), speculative_num_steps=speculative_num_steps, + is_eagle_draft_decode=is_eagle_draft_decode, ) diff --git a/test/registered/kv_canary/test_self_unit_runner_health.py b/test/registered/kv_canary/test_self_unit_runner_health.py index 1ee396ea47..1e591da5b9 100644 --- a/test/registered/kv_canary/test_self_unit_runner_health.py +++ b/test/registered/kv_canary/test_self_unit_runner_health.py @@ -54,6 +54,36 @@ class TestSelfUnitManagerHealth(CanaryManagerTestCase): manager._outer_step_counter = 2000 manager._health_checker.step() + def test_kernel_run_counter_watchdog_expects_sweep_on_target_only( + self, + ) -> None: + """The scheduler attaches the radix cache only to the target runner: + a draft runner never sweeps, while a target whose sweep never launches + must still fail the watchdog.""" + + def check_twice(*, is_draft: bool) -> None: + manager = make_manager( + device=self.device, + config=make_config(sweep_interval=100), + is_eagle_draft_decode=is_draft, + ) + counters = manager._device_state.kernel_run_counters + counters.zero_() + for step in (1000, 2000): + for tag in ( + CanaryLaunchTag.HEAD_K_FULL, + CanaryLaunchTag.HEAD_V_FULL, + CanaryLaunchTag.TAIL_K_FULL, + CanaryLaunchTag.TAIL_V_FULL, + ): + counters[tag.value] += 1 + manager._outer_step_counter = step + manager._health_checker.step() + + check_twice(is_draft=True) + with self.assertRaisesRegex(RuntimeError, "SWEEP_K_FULL"): + check_twice(is_draft=False) + def test_periodic_stats_log_every_n_step(self) -> None: """Verify periodic stats are logged at the configured interval.""" config = make_config(stats_print_every_n_steps=5) @@ -85,6 +115,7 @@ class TestKernelRunCounterDeltaCheck(CustomTestCase): device_state.kernel_run_counters = torch.zeros(num_tags, dtype=torch.int64) return KernelRunCounterHealthChecker( config=config, + is_draft_runner=False, device_state=device_state, active_tags=active_tags, outer_step_counter_getter=lambda: outer_step,