Move on_idle from runtime_checker mixin into Scheduler (#25617)
This commit is contained in:
@@ -3031,6 +3031,37 @@ class Scheduler(
|
|||||||
if_success = False
|
if_success = False
|
||||||
return ClearHiCacheReqOutput(success=if_success)
|
return ClearHiCacheReqOutput(success=if_success)
|
||||||
|
|
||||||
|
def on_idle(self):
|
||||||
|
"""Idle housekeeping: guard, check, metrics, reset, sleep."""
|
||||||
|
if not self.is_fully_idle():
|
||||||
|
return
|
||||||
|
|
||||||
|
# memory leak check (skipped for hisparse — pool counters intentionally
|
||||||
|
# diverge during host-backup, see _get_swa_token_info clamp).
|
||||||
|
if not self.enable_hisparse:
|
||||||
|
has_leak, messages = self._check_all_pools(self.get_pool_stats())
|
||||||
|
if has_leak:
|
||||||
|
self._report_leak("pool", "\n".join(messages))
|
||||||
|
self._check_req_pool()
|
||||||
|
|
||||||
|
# tree cache sanity check
|
||||||
|
self._check_tree_cache()
|
||||||
|
|
||||||
|
# metrics every 30s
|
||||||
|
self._maybe_log_idle_metrics()
|
||||||
|
|
||||||
|
# kv event publishing
|
||||||
|
self._publish_kv_events()
|
||||||
|
|
||||||
|
# reset token ratio
|
||||||
|
self.new_token_ratio = self.init_new_token_ratio
|
||||||
|
|
||||||
|
# reset device timer window so idle time isn't counted
|
||||||
|
self.reset_device_timer_window()
|
||||||
|
|
||||||
|
# sleep until next event
|
||||||
|
self.maybe_sleep_on_idle()
|
||||||
|
|
||||||
def is_fully_idle(self, for_health_check=False) -> bool:
|
def is_fully_idle(self, for_health_check=False) -> bool:
|
||||||
# Health check piggybacks on running requests in process_output.
|
# Health check piggybacks on running requests in process_output.
|
||||||
# Only running_batch + waiting_queue guarantee active GPU processing;
|
# Only running_batch + waiting_queue guarantee active GPU processing;
|
||||||
|
|||||||
@@ -548,37 +548,6 @@ class SchedulerRuntimeCheckerMixin:
|
|||||||
):
|
):
|
||||||
self.tree_cache.sanity_check()
|
self.tree_cache.sanity_check()
|
||||||
|
|
||||||
def on_idle(self: Scheduler):
|
|
||||||
"""Idle housekeeping: guard, check, metrics, reset, sleep."""
|
|
||||||
if not self.is_fully_idle():
|
|
||||||
return
|
|
||||||
|
|
||||||
# memory leak check (skipped for hisparse — pool counters intentionally
|
|
||||||
# diverge during host-backup, see _get_swa_token_info clamp).
|
|
||||||
if not self.enable_hisparse:
|
|
||||||
has_leak, messages = self._check_all_pools(self.get_pool_stats())
|
|
||||||
if has_leak:
|
|
||||||
self._report_leak("pool", "\n".join(messages))
|
|
||||||
self._check_req_pool()
|
|
||||||
|
|
||||||
# tree cache sanity check
|
|
||||||
self._check_tree_cache()
|
|
||||||
|
|
||||||
# metrics every 30s
|
|
||||||
self._maybe_log_idle_metrics()
|
|
||||||
|
|
||||||
# kv event publishing
|
|
||||||
self._publish_kv_events()
|
|
||||||
|
|
||||||
# reset token ratio
|
|
||||||
self.new_token_ratio = self.init_new_token_ratio
|
|
||||||
|
|
||||||
# reset device timer window so idle time isn't counted
|
|
||||||
self.reset_device_timer_window()
|
|
||||||
|
|
||||||
# sleep until next event
|
|
||||||
self.maybe_sleep_on_idle()
|
|
||||||
|
|
||||||
|
|
||||||
def create_scheduler_watchdog(
|
def create_scheduler_watchdog(
|
||||||
scheduler: Scheduler, watchdog_timeout: float, soft: bool = False
|
scheduler: Scheduler, watchdog_timeout: float, soft: bool = False
|
||||||
|
|||||||
Reference in New Issue
Block a user