Lift forward_ct/cur_batch and use direct access in watchdog (#25442)
This commit is contained in:
@@ -349,6 +349,9 @@ class Scheduler(
|
||||
dp_rank: Optional[int],
|
||||
):
|
||||
self.is_initializing = True
|
||||
# init_soft_watchdog starts a daemon thread that reads these on its first tick.
|
||||
self.forward_ct: int = 0
|
||||
self.cur_batch: Optional[ScheduleBatch] = None
|
||||
self.init_soft_watchdog(server_args)
|
||||
|
||||
# Parse args
|
||||
|
||||
@@ -594,9 +594,8 @@ def create_scheduler_watchdog(
|
||||
|
||||
return WatchdogRaw(
|
||||
debug_name="Scheduler",
|
||||
get_counter=lambda: getattr(scheduler, "forward_ct", 0),
|
||||
is_active=lambda: scheduler.is_initializing
|
||||
or getattr(scheduler, "cur_batch", None) is not None,
|
||||
get_counter=lambda: scheduler.forward_ct,
|
||||
is_active=lambda: scheduler.is_initializing or scheduler.cur_batch is not None,
|
||||
watchdog_timeout=watchdog_timeout,
|
||||
soft=soft,
|
||||
dump_info=dump_info,
|
||||
|
||||
Reference in New Issue
Block a user